From 6afe67afa0f4ec3e171b77082a580d5bbbec64ef Mon Sep 17 00:00:00 2001 From: adam-m Date: Fri, 6 Jan 2012 15:06:19 -0500 Subject: [PATCH] In addition to standard encoding of queries before sending over net, perform escaping of Lucene special characters to always treat Solr queries as words. This improves files query accuracy and highlighting dramatically and we assume user has no Lucene knowledge anyways. --- .../HighlightedMatchesSource.java | 2 +- .../KeywordSearchResultFactory.java | 2 +- .../keywordsearch/KeywordSearchUtil.java | 27 +++++++++++++++++-- .../autopsy/keywordsearch/LuceneQuery.java | 2 +- .../keywordsearch/TermComponentQuery.java | 9 +++++-- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java index 0699d61ac7..dbd3e41f80 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java @@ -57,7 +57,7 @@ class HighlightedMatchesSource implements MarkupSource { public String getMarkup() { SolrQuery q = new SolrQuery(); - final String queryEscaped = KeywordSearchUtil.escapeLuceneQuery(solrQuery); + final String queryEscaped = KeywordSearchUtil.escapeLuceneQuery(solrQuery, true); q.setQuery(queryEscaped); q.addFilterQuery("id:" + content.getId()); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java index af06e64f36..7665305c20 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java @@ -185,7 +185,7 @@ public class KeywordSearchResultFactory extends ChildFactory { StringBuilder highlightQuery = new StringBuilder(); Collection terms = tcq.getTerms(); for (Term term : terms) { - final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); + final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm(), true); highlightQuery.append(termS); highlightQuery.append(" "); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java index efd4b7e545..9b8afa65a1 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java @@ -58,10 +58,32 @@ public class KeywordSearchUtil { return dirName; } - public static String escapeLuceneQuery(String query) { + /** + * Perform standard escaping / encoding into UTF-8 before sending over net + * @param query to be encoded + * @param escapeLuceneChars if true perform first escaping of Lucene specific special chars + * such as /+-&|!(){}[]^"~*?:\ and treat the whole query as literal word + * @return encoded query + */ + public static String escapeLuceneQuery(String query, boolean escapeLuceneChars) { String queryEscaped = null; + String inputString = query; + + if (escapeLuceneChars == true) { + final String ESCAPE_CHARS = "/+-&|!(){}[]^\"~*?:\\"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i< inputString.length(); ++i) { + char c = inputString.charAt(i); + if (ESCAPE_CHARS.contains(Character.toString(c)) ) { + sb.append("\\"); + } + sb.append(c); + } + inputString = sb.toString(); + } + try { - queryEscaped = URLEncoder.encode(query, "UTF-8"); + queryEscaped = URLEncoder.encode(inputString, "UTF-8"); } catch (UnsupportedEncodingException ex) { logger.log(Level.SEVERE, "Error escaping URL query, should not happen.", ex); @@ -70,6 +92,7 @@ public class KeywordSearchUtil { return queryEscaped; } + public static void displayDialog(final String title, final String message, final DIALOG_MESSAGE_TYPE type) { int messageType; if (type == DIALOG_MESSAGE_TYPE.ERROR) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 7060b7c780..82d3799523 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -56,7 +56,7 @@ public class LuceneQuery implements KeywordSearchQuery { @Override public void escape() { - queryEscaped = KeywordSearchUtil.escapeLuceneQuery(query); + queryEscaped = KeywordSearchUtil.escapeLuceneQuery(query, true); isEscaped = true; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index ffda074aaf..d25a8c1dd9 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -165,11 +165,15 @@ public class TermComponentQuery implements KeywordSearchQuery { //it's much more efficient and should yield the same file IDs as per match queries //requires http POST query method due to potentially large query size StringBuilder filesQueryB = new StringBuilder(); + final int lastTerm = terms.size() -1; + int curTerm = 0; for (Term term : terms) { - //final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); + //final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm(), true); final String termS = term.getTerm(); filesQueryB.append(termS); - filesQueryB.append(" "); + if (curTerm != lastTerm) + filesQueryB.append(" "); + ++curTerm; } List uniqueMatches = new ArrayList(); @@ -194,6 +198,7 @@ public class TermComponentQuery implements KeywordSearchQuery { results.add(f); } } + return results;