From 27f31446c07b6012f5e398b7fe061ce81800541b Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Mon, 30 Apr 2018 17:09:55 +0200 Subject: [PATCH] search term parsing + highlighting --- .../persistance/DatabaseBackend.java | 10 +- .../services/MessageSearchTask.java | 6 +- .../services/XmppConnectionService.java | 2 +- .../conversations/ui/SearchActivity.java | 16 ++-- .../ui/adapter/MessageAdapter.java | 11 +-- .../interfaces/OnSearchResultsAvailable.java | 2 +- .../siacs/conversations/utils/FtsUtils.java | 94 +++++++++++++++++++ .../conversations/utils/StylingHelper.java | 10 +- 8 files changed, 128 insertions(+), 23 deletions(-) create mode 100644 src/main/java/eu/siacs/conversations/utils/FtsUtils.java diff --git a/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java b/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java index 2d4f431c3..e42de05e6 100644 --- a/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java +++ b/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java @@ -50,6 +50,7 @@ import eu.siacs.conversations.entities.Roster; import eu.siacs.conversations.entities.ServiceDiscoveryResult; import eu.siacs.conversations.services.ShortcutService; import eu.siacs.conversations.utils.CryptoHelper; +import eu.siacs.conversations.utils.FtsUtils; import eu.siacs.conversations.utils.MimeUtils; import eu.siacs.conversations.utils.Resolver; import eu.siacs.conversations.xmpp.mam.MamReference; @@ -229,6 +230,10 @@ public class DatabaseBackend extends SQLiteOpenHelper { db.execSQL(CREATE_IDENTITIES_STATEMENT); db.execSQL(CREATE_PRESENCE_TEMPLATES_STATEMENT); db.execSQL(CREATE_RESOLVER_RESULTS_TABLE); + db.execSQL(CREATE_MESSAGE_INDEX_TABLE); + db.execSQL(CREATE_MESSAGE_INSERT_TRIGGER); + db.execSQL(CREATE_MESSAGE_UPDATE_TRIGGER); + db.execSQL(CREATE_MESSAGE_DELETE_TRIGGER); } @Override @@ -718,10 +723,11 @@ public class DatabaseBackend extends SQLiteOpenHelper { return list; } - public Cursor getMessageSearchCursor(String term) { + public Cursor getMessageSearchCursor(List term) { SQLiteDatabase db = this.getReadableDatabase(); String SQL = "SELECT "+Message.TABLENAME+".*,"+Conversation.TABLENAME+'.'+Conversation.CONTACTJID+','+Conversation.TABLENAME+'.'+Conversation.ACCOUNT+','+Conversation.TABLENAME+'.'+Conversation.MODE+" FROM "+Message.TABLENAME +" join "+Conversation.TABLENAME+" on "+Message.TABLENAME+'.'+Message.CONVERSATION+'='+Conversation.TABLENAME+'.'+Conversation.UUID+" join messages_index ON messages_index.uuid=messages.uuid where "+Message.ENCRYPTION+" NOT IN("+Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE+','+Message.ENCRYPTION_PGP+','+Message.ENCRYPTION_DECRYPTION_FAILED+") AND messages_index.body MATCH ? ORDER BY "+Message.TIME_SENT+" DESC limit "+Config.MAX_SEARCH_RESULTS; - return db.rawQuery(SQL,new String[]{'%'+term+'%'}); + Log.d(Config.LOGTAG,"search term: "+FtsUtils.toMatchString(term)); + return db.rawQuery(SQL,new String[]{FtsUtils.toMatchString(term)}); } public Iterable getMessagesIterable(final Conversation conversation) { diff --git a/src/main/java/eu/siacs/conversations/services/MessageSearchTask.java b/src/main/java/eu/siacs/conversations/services/MessageSearchTask.java index 416e54b9b..5d5aeefbe 100644 --- a/src/main/java/eu/siacs/conversations/services/MessageSearchTask.java +++ b/src/main/java/eu/siacs/conversations/services/MessageSearchTask.java @@ -55,18 +55,18 @@ public class MessageSearchTask implements Runnable, Cancellable { private static final ReplacingSerialSingleThreadExecutor EXECUTOR = new ReplacingSerialSingleThreadExecutor(MessageSearchTask.class.getName()); private final XmppConnectionService xmppConnectionService; - private final String term; + private final List term; private final OnSearchResultsAvailable onSearchResultsAvailable; private boolean isCancelled = false; - private MessageSearchTask(XmppConnectionService xmppConnectionService, String term, OnSearchResultsAvailable onSearchResultsAvailable) { + private MessageSearchTask(XmppConnectionService xmppConnectionService, List term, OnSearchResultsAvailable onSearchResultsAvailable) { this.xmppConnectionService = xmppConnectionService; this.term = term; this.onSearchResultsAvailable = onSearchResultsAvailable; } - public static void search(XmppConnectionService xmppConnectionService, String term, OnSearchResultsAvailable onSearchResultsAvailable) { + public static void search(XmppConnectionService xmppConnectionService, List term, OnSearchResultsAvailable onSearchResultsAvailable) { new MessageSearchTask(xmppConnectionService, term, onSearchResultsAvailable).executeInBackground(); } diff --git a/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java b/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java index de1909b74..92e413c42 100644 --- a/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java +++ b/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java @@ -535,7 +535,7 @@ public class XmppConnectionService extends Service { return find(getConversations(), account, jid); } - public void search(String term, OnSearchResultsAvailable onSearchResultsAvailable) { + public void search(List term, OnSearchResultsAvailable onSearchResultsAvailable) { MessageSearchTask.search(this, term, onSearchResultsAvailable); } diff --git a/src/main/java/eu/siacs/conversations/ui/SearchActivity.java b/src/main/java/eu/siacs/conversations/ui/SearchActivity.java index 784394deb..83f80d3fc 100644 --- a/src/main/java/eu/siacs/conversations/ui/SearchActivity.java +++ b/src/main/java/eu/siacs/conversations/ui/SearchActivity.java @@ -64,6 +64,7 @@ import eu.siacs.conversations.ui.util.DateSeparator; import eu.siacs.conversations.ui.util.Drawable; import eu.siacs.conversations.ui.util.ListViewUtils; import eu.siacs.conversations.ui.util.ShareUtil; +import eu.siacs.conversations.utils.FtsUtils; import eu.siacs.conversations.utils.MessageUtils; import static eu.siacs.conversations.ui.util.SoftKeyboardUtils.hideSoftKeyboard; @@ -75,7 +76,7 @@ public class SearchActivity extends XmppActivity implements TextWatcher, OnSearc private MessageAdapter messageListAdapter; private final List messages = new ArrayList<>(); private WeakReference selectedMessageReference = new WeakReference<>(null); - private final ChangeWatcher currentSearch = new ChangeWatcher<>(); + private final ChangeWatcher> currentSearch = new ChangeWatcher<>(); @Override public void onCreate(final Bundle savedInstanceState) { @@ -153,13 +154,10 @@ public class SearchActivity extends XmppActivity implements TextWatcher, OnSearc } private void quote(Message message) { - String text = MessageUtils.prepareQuote(message); - final Conversational conversational = message.getConversation(); - switchToConversationAndQuote(wrap(message.getConversation()), text); + switchToConversationAndQuote(wrap(message.getConversation()), MessageUtils.prepareQuote(message)); } private Conversation wrap(Conversational conversational) { - final Conversation conversation; if (conversational instanceof Conversation) { return (Conversation) conversational; } else { @@ -205,12 +203,12 @@ public class SearchActivity extends XmppActivity implements TextWatcher, OnSearc @Override public void afterTextChanged(Editable s) { - final String term = s.toString().trim(); + final List term = FtsUtils.parse(s.toString().trim()); if (!currentSearch.watch(term)) { return; } - if (term.length() > 0) { - xmppConnectionService.search(s.toString().trim(), this); + if (term.size() > 0) { + xmppConnectionService.search(term, this); } else { MessageSearchTask.cancelRunningTasks(); this.messages.clear(); @@ -221,7 +219,7 @@ public class SearchActivity extends XmppActivity implements TextWatcher, OnSearc } @Override - public void onSearchResultsAvailable(String term, List messages) { + public void onSearchResultsAvailable(List term, List messages) { runOnUiThread(() -> { this.messages.clear(); messageListAdapter.setHighlightedTerm(term); diff --git a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java index 3d7e069f7..77dac289e 100644 --- a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java +++ b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java @@ -34,7 +34,6 @@ import android.view.ActionMode; import android.view.Menu; import android.view.MenuItem; import android.view.View; -import android.view.View.OnClickListener; import android.view.ViewGroup; import android.view.WindowManager; import android.widget.ArrayAdapter; @@ -99,7 +98,7 @@ public class MessageAdapter extends ArrayAdapter implements CopyTextVie + "\\;\\/\\?\\@\\&\\=\\#\\~\\-\\.\\+\\!\\*\\'\\(\\)\\,\\_])" + "|(?:\\%[a-fA-F0-9]{2}))+"); - private String highlightedText = null; + private List highlightedTerm = null; private static final Linkify.TransformFilter WEBURL_TRANSFORM_FILTER = (matcher, url) -> { if (url == null) { @@ -550,8 +549,8 @@ public class MessageAdapter extends ArrayAdapter implements CopyTextVie } StylingHelper.format(body, viewHolder.messageBody.getCurrentTextColor()); - if (highlightedText != null) { - StylingHelper.highlight(activity, body, highlightedText, StylingHelper.isDarkText(viewHolder.messageBody)); + if (highlightedTerm != null) { + StylingHelper.highlight(activity, body, highlightedTerm, StylingHelper.isDarkText(viewHolder.messageBody)); } Linkify.addLinks(body, XMPP_PATTERN, "xmpp", XMPPURI_MATCH_FILTER, null); @@ -1008,8 +1007,8 @@ public class MessageAdapter extends ArrayAdapter implements CopyTextVie } } - public void setHighlightedTerm(String term) { - this.highlightedText = term; + public void setHighlightedTerm(List term) { + this.highlightedTerm = term; } public interface OnQuoteListener { diff --git a/src/main/java/eu/siacs/conversations/ui/interfaces/OnSearchResultsAvailable.java b/src/main/java/eu/siacs/conversations/ui/interfaces/OnSearchResultsAvailable.java index d884af361..2dcdff86f 100644 --- a/src/main/java/eu/siacs/conversations/ui/interfaces/OnSearchResultsAvailable.java +++ b/src/main/java/eu/siacs/conversations/ui/interfaces/OnSearchResultsAvailable.java @@ -35,6 +35,6 @@ import eu.siacs.conversations.entities.Message; public interface OnSearchResultsAvailable { - void onSearchResultsAvailable(String term, List messages); + void onSearchResultsAvailable(List term, List messages); } diff --git a/src/main/java/eu/siacs/conversations/utils/FtsUtils.java b/src/main/java/eu/siacs/conversations/utils/FtsUtils.java new file mode 100644 index 000000000..46c400280 --- /dev/null +++ b/src/main/java/eu/siacs/conversations/utils/FtsUtils.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2018, Daniel Gultsch All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package eu.siacs.conversations.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +public class FtsUtils { + + private static List KEYWORDS = Arrays.asList("OR", "AND"); + + public static List parse(String input) { + List term = new ArrayList<>(); + for (String part : input.split("\\s+")) { + if (part.isEmpty()) { + continue; + } + final String cleaned = part.substring(getStartIndex(part), getEndIndex(part) +1); + if (isKeyword(cleaned)) { + term.add(part); + } else { + term.add(cleaned); + } + } + return term; + } + + public static String toMatchString(List terms) { + StringBuilder builder = new StringBuilder(); + for (String term : terms) { + if (builder.length() != 0) { + builder.append(' '); + } + if (isKeyword(term)) { + builder.append(term.toUpperCase(Locale.ENGLISH)); + } else if (term.contains("*") || term.startsWith("-")) { + builder.append(term); + } else { + builder.append('*').append(term).append('*'); + } + } + return builder.toString(); + } + + public static boolean isKeyword(String term) { + return KEYWORDS.contains(term.toUpperCase(Locale.ENGLISH)); + } + + private static int getStartIndex(String term) { + int index = 0; + while (term.charAt(index) == '*') { + ++index; + } + return index; + } + + private static int getEndIndex(String term) { + int index = term.length() - 1; + while (term.charAt(index) == '*') { + --index; + } + return index; + } + +} diff --git a/src/main/java/eu/siacs/conversations/utils/StylingHelper.java b/src/main/java/eu/siacs/conversations/utils/StylingHelper.java index 06dfaef90..23e6709f7 100644 --- a/src/main/java/eu/siacs/conversations/utils/StylingHelper.java +++ b/src/main/java/eu/siacs/conversations/utils/StylingHelper.java @@ -91,7 +91,15 @@ public class StylingHelper { format(editable, end, editable.length() - 1, textColor); } - public static void highlight(final Context context, final Editable editable, String needle, boolean dark) { + public static void highlight(final Context context, final Editable editable, List needles, boolean dark) { + for(String needle : needles) { + if (!FtsUtils.isKeyword(needle)) { + highlight(context, editable, needle, dark); + } + } + } + + private static void highlight(final Context context, final Editable editable, String needle, boolean dark) { final int length = needle.length(); String string = editable.toString(); int start = indexOfIgnoreCase(string, needle, 0);