Revert "Fixes: commons-app#3179 Make category search non case-sensitive (commons-app#3326)" (commons-app#3636)

Simply lower casing the name of the category sent to the server
doesn't result in the server doing a case insensitive category
search. In fact, it reduces the category search space as only
categories that has a lower case character is searched even
if the search text contains upper case characters.

The test case did not catch this issue as the first character
of the title is case insensitive[1].

So, revert the changes done in commit afdeaae.

See further disucssion in the issue thread of commons-app#3179 starting
from [2].

[1]: https://www.mediawiki.org/wiki/Manual:Page_title
[2]: commons-app#3179 (comment)

Author: sivaraam

app/src/main/java/fr/free/nrw/commons/category/CategoriesModel.java

@@ -0,0 +1,260 @@
+package fr.free.nrw.commons.category;
+
+import android.text.TextUtils;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import fr.free.nrw.commons.kvstore.JsonKvStore;
+import fr.free.nrw.commons.upload.GpsCategoryModel;
+import fr.free.nrw.commons.utils.StringSortingUtils;
+import io.reactivex.Observable;
+import timber.log.Timber;
+
+/**
+ * The model class for categories in upload
+ */
+public class CategoriesModel{
+    private static final int SEARCH_CATS_LIMIT = 25;
+
+    private final CategoryClient categoryClient;
+    private final CategoryDao categoryDao;
+    private final JsonKvStore directKvStore;
+
+    private HashMap<String, ArrayList<String>> categoriesCache;
+    private List<CategoryItem> selectedCategories;
+
+    @Inject GpsCategoryModel gpsCategoryModel;
+    @Inject
+    public CategoriesModel(CategoryClient categoryClient,
+                           CategoryDao categoryDao,
+                           @Named("default_preferences") JsonKvStore directKvStore) {
+        this.categoryClient = categoryClient;
+        this.categoryDao = categoryDao;
+        this.directKvStore = directKvStore;
+        this.categoriesCache = new HashMap<>();
+        this.selectedCategories = new ArrayList<>();
+    }
+
+    /**
+     * Sorts CategoryItem by similarity
+     * @param filter
+     * @return
+     */
+    public Comparator<CategoryItem> sortBySimilarity(final String filter) {
+        Comparator<String> stringSimilarityComparator = StringSortingUtils.sortBySimilarity(filter);
+        return (firstItem, secondItem) -> stringSimilarityComparator
+                .compare(firstItem.getName(), secondItem.getName());
+    }
+
+    /**
+     * Returns if the item contains an year
+     * @param item
+     * @return
+     */
+    public boolean containsYear(String item) {
+        //Check for current and previous year to exclude these categories from removal
+        Calendar now = Calendar.getInstance();
+        int year = now.get(Calendar.YEAR);
+        String yearInString = String.valueOf(year);
+
+        int prevYear = year - 1;
+        String prevYearInString = String.valueOf(prevYear);
+        Timber.d("Previous year: %s", prevYearInString);
+
+        //Check if item contains a 4-digit word anywhere within the string (.* is wildcard)
+        //And that item does not equal the current year or previous year
+        //And if it is an irrelevant category such as Media_needing_categories_as_of_16_June_2017(Issue #750)
+        //Check if the year in the form of XX(X)0s is relevant, i.e. in the 2000s or 2010s as stated in Issue #1029
+        return ((item.matches(".*(19|20)\\d{2}.*") && !item.contains(yearInString) && !item.contains(prevYearInString))
+                || item.matches("(.*)needing(.*)") || item.matches("(.*)taken on(.*)")
+                || (item.matches(".*0s.*") && !item.matches(".*(200|201)0s.*")));
+    }
+
+    /**
+     * Updates category count in category dao
+     * @param item
+     */
+    public void updateCategoryCount(CategoryItem item) {
+        Category category = categoryDao.find(item.getName());
+
+        // Newly used category...
+        if (category == null) {
+            category = new Category(null, item.getName(), new Date(), 0);
+        }
+
+        category.incTimesUsed();
+        categoryDao.save(category);
+    }
+
+    boolean cacheContainsKey(String term) {
+        return categoriesCache.containsKey(term);
+    }
+    //endregion
+
+    /**
+     * Regional category search
+     * @param term
+     * @param imageTitleList
+     * @return
+     */
+    public Observable<CategoryItem> searchAll(String term, List<String> imageTitleList) {
+        //If query text is empty, show him category based on gps and title and recent searches
+        if (TextUtils.isEmpty(term)) {
+            Observable<CategoryItem> categoryItemObservable = gpsCategories()
+                    .concatWith(titleCategories(imageTitleList));
+            if (hasDirectCategories()) {
+                categoryItemObservable.concatWith(directCategories().concatWith(recentCategories()));
+            }
+            return categoryItemObservable;
+        }
+
+        //if user types in something that is in cache, return cached category
+        if (cacheContainsKey(term)) {
+            return Observable.fromIterable(getCachedCategories(term))
+                    .map(name -> new CategoryItem(name, false));
+        }
+
+        //otherwise, search API for matching categories
+        return categoryClient
+                .searchCategoriesForPrefix(term, SEARCH_CATS_LIMIT)
+                .map(name -> new CategoryItem(name, false));
+    }
+
+
+    /**
+     * Returns cached categories
+     * @param term
+     * @return
+     */
+    private ArrayList<String> getCachedCategories(String term) {
+        return categoriesCache.get(term);
+    }
+
+    /**
+     * Returns if we have a category in DirectKV Store
+     * @return
+     */
+    private boolean hasDirectCategories() {
+        return !directKvStore.getString("Category", "").equals("");
+    }
+
+    /**
+     * Returns categories in DirectKVStore
+     * @return
+     */
+    private Observable<CategoryItem> directCategories() {
+        String directCategory = directKvStore.getString("Category", "");
+        List<String> categoryList = new ArrayList<>();
+        Timber.d("Direct category found: " + directCategory);
+
+        if (!directCategory.equals("")) {
+            categoryList.add(directCategory);
+            Timber.d("DirectCat does not equal emptyString. Direct Cat list has " + categoryList);
+        }
+        return Observable.fromIterable(categoryList).map(name -> new CategoryItem(name, false));
+    }
+
+    /**
+     * Returns GPS categories
+     * @return
+     */
+    Observable<CategoryItem> gpsCategories() {
+        return Observable.fromIterable(gpsCategoryModel.getCategoryList())
+                .map(name -> new CategoryItem(name, false));
+    }
+
+    /**
+     * Returns title based categories
+     * @param titleList
+     * @return
+     */
+    private Observable<CategoryItem> titleCategories(List<String> titleList) {
+        return Observable.fromIterable(titleList)
+                .concatMap(this::getTitleCategories);
+    }
+
+    /**
+     * Return category for single title
+     * @param title
+     * @return
+     */
+    private Observable<CategoryItem> getTitleCategories(String title) {
+        return categoryClient.searchCategories(title, SEARCH_CATS_LIMIT)
+                .map(name -> new CategoryItem(name, false));
+    }
+
+    /**
+     * Returns recent categories
+     * @return
+     */
+    private Observable<CategoryItem> recentCategories() {
+        return Observable.fromIterable(categoryDao.recentCategories(SEARCH_CATS_LIMIT))
+                .map(s -> new CategoryItem(s, false));
+    }
+
+    /**
+     * Handles category item selection
+     * @param item
+     */
+    public void onCategoryItemClicked(CategoryItem item) {
+        if (item.isSelected()) {
+            selectCategory(item);
+            updateCategoryCount(item);
+        } else {
+            unselectCategory(item);
+        }
+    }
+
+    /**
+     * Select's category
+     * @param item
+     */
+    public void selectCategory(CategoryItem item) {
+        selectedCategories.add(item);
+    }
+
+    /**
+     * Unselect Category
+     * @param item
+     */
+    public void unselectCategory(CategoryItem item) {
+        selectedCategories.remove(item);
+    }
+
+
+    /**
+     * Get Selected Categories
+     * @return
+     */
+    public List<CategoryItem> getSelectedCategories() {
+        return selectedCategories;
+    }
+
+    /**
+     * Get Categories String List
+     * @return
+     */
+    public List<String> getCategoryStringList() {
+        List<String> output = new ArrayList<>();
+        for (CategoryItem item : selectedCategories) {
+            output.add(item.getName());
+        }
+        return output;
+    }
+
+    /**
+     * Cleanup the existing in memory cache's
+     */
+    public void cleanUp() {
+        this.categoriesCache.clear();
+        this.selectedCategories.clear();
+    }
+}

app/src/main/java/fr/free/nrw/commons/category/CategoryClient.java

@@ -0,0 +1,125 @@
+package fr.free.nrw.commons.category;
+
+
+import androidx.annotation.NonNull;
+
+import org.wikipedia.dataclient.mwapi.MwQueryPage;
+import org.wikipedia.dataclient.mwapi.MwQueryResponse;
+import org.wikipedia.dataclient.mwapi.MwQueryResult;
+
+import java.util.List;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import io.reactivex.Observable;
+import timber.log.Timber;
+
+/**
+ * Category Client to handle custom calls to Commons MediaWiki APIs
+ */
+@Singleton
+public class CategoryClient {
+
+    private final CategoryInterface CategoryInterface;
+
+    @Inject
+    public CategoryClient(CategoryInterface CategoryInterface) {
+        this.CategoryInterface = CategoryInterface;
+    }
+
+    /**
+     * Searches for categories containing the specified string.
+     *
+     * @param filter    The string to be searched
+     * @param itemLimit How many results are returned
+     * @param offset    Starts returning items from the nth result. If offset is 9, the response starts with the 9th item of the search result
+     * @return
+     */
+    public Observable<String> searchCategories(String filter, int itemLimit, int offset) {
+        return responseToCategoryName(CategoryInterface.searchCategories(filter, itemLimit, offset));
+
+    }
+
+    /**
+     * Searches for categories containing the specified string.
+     *
+     * @param filter    The string to be searched
+     * @param itemLimit How many results are returned
+     * @return
+     */
+    public Observable<String> searchCategories(String filter, int itemLimit) {
+        return searchCategories(filter, itemLimit, 0);
+
+    }
+
+    /**
+     * Searches for categories starting with the specified string.
+     *
+     * @param prefix    The prefix to be searched
+     * @param itemLimit How many results are returned
+     * @param offset    Starts returning items from the nth result. If offset is 9, the response starts with the 9th item of the search result
+     * @return
+     */
+    public Observable<String> searchCategoriesForPrefix(String prefix, int itemLimit, int offset) {
+        return responseToCategoryName(CategoryInterface.searchCategoriesForPrefix(prefix, itemLimit, offset));
+    }
+
+    /**
+     * Searches for categories starting with the specified string.
+     *
+     * @param prefix    The prefix to be searched
+     * @param itemLimit How many results are returned
+     * @return
+     */
+    public Observable<String> searchCategoriesForPrefix(String prefix, int itemLimit) {
+        return searchCategoriesForPrefix(prefix, itemLimit, 0);
+    }
+
+
+    /**
+     * The method takes categoryName as input and returns a List of Subcategories
+     * It uses the generator query API to get the subcategories in a category, 500 at a time.
+     *
+     * @param categoryName Category name as defined on commons
+     * @return Observable emitting the categories returned. If our search yielded "Category:Test", "Test" is emitted.
+     */
+    public Observable<String> getSubCategoryList(String categoryName) {
+        return responseToCategoryName(CategoryInterface.getSubCategoryList(categoryName));
+    }
+
+    /**
+     * The method takes categoryName as input and returns a List of parent categories
+     * It uses the generator query API to get the parent categories of a category, 500 at a time.
+     *
+     * @param categoryName Category name as defined on commons
+     * @return
+     */
+    @NonNull
+    public Observable<String> getParentCategoryList(String categoryName) {
+        return responseToCategoryName(CategoryInterface.getParentCategoryList(categoryName));
+    }
+
+    /**
+     * Internal function to reduce code reuse. Extracts the categories returned from MwQueryResponse.
+     *
+     * @param responseObservable The query response observable
+     * @return Observable emitting the categories returned. If our search yielded "Category:Test", "Test" is emitted.
+     */
+    private Observable<String> responseToCategoryName(Observable<MwQueryResponse> responseObservable) {
+        return responseObservable
+                .flatMap(mwQueryResponse -> {
+                    MwQueryResult query;
+                    List<MwQueryPage> pages;
+                    if ((query = mwQueryResponse.query()) == null ||
+                            (pages = query.pages()) == null) {
+                        Timber.d("No categories returned.");
+                        return Observable.empty();
+                    } else
+                        return Observable.fromIterable(pages);
+                })
+                .map(MwQueryPage::title)
+                .doOnEach(s -> Timber.d("Category returned: %s", s))
+                .map(cat -> cat.replace("Category:", ""));
+    }
+}