diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java index 5666332ce1..58a48b865c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java @@ -141,7 +141,7 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup { Keyword keywordQuery = new Keyword(queryStr, false); chunksQuery = new LuceneQuery(keywordQuery); KeywordQueryFilter contentIdFilter = new KeywordQueryFilter(FilterType.CHUNK, contentId); - chunksQuery.setFilter(contentIdFilter); + chunksQuery.addFilter(contentIdFilter); try { hits = chunksQuery.performQuery(); } catch (NoOpenCoreException ex) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java index c87244bbf5..a62b884ae8 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java @@ -16,43 +16,63 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.keywordsearch; /** * - * Filter to select only specific id or chunks for that id + * Filter to restrict query only specific files, chunks, images + * Single filter supports multiple ids per file/chunk/image, that act as OR filter */ public class KeywordQueryFilter { - public static enum FilterType {FILE, CHUNK}; - private long idFilter; + + public static enum FilterType { + + FILE, CHUNK, IMAGE + }; + private long[] idFilters; private FilterType filterType; - + public KeywordQueryFilter(FilterType filterType, long id) { this.filterType = filterType; - this.idFilter = id; + this.idFilters = new long[1]; + this.idFilters[0] = id; } - - public long getIdFilter() { - return idFilter; + + public KeywordQueryFilter(FilterType filterType, long[] ids) { + this.filterType = filterType; + this.idFilters = ids; } + + public long[] getIdFilters() { + return idFilters; + } + public FilterType getFilterType() { return filterType; } - + @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(Server.Schema.ID.toString()); - sb.append(":"); - sb.append(Long.toString(idFilter)); - if (filterType == FilterType.CHUNK) { - sb.append("_*"); + String id = null; + for (int i = 0; i < idFilters.length; ++i) { + if (i > 0) { + sb.append(" "); //OR + } + long idVal = idFilters[i]; + if (filterType == FilterType.IMAGE) { + id = Server.Schema.IMAGE_ID.toString(); + } else { + id = Server.Schema.ID.toString(); + } + sb.append(id); + sb.append(":"); + sb.append(Long.toString(idVal)); + if (filterType == FilterType.CHUNK) { + sb.append("_*"); + } } + return sb.toString(); } - - - - } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 40591e0487..4d5d2f0292 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -20,11 +20,15 @@ package org.sleuthkit.autopsy.keywordsearch; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; +import java.lang.Long; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -57,6 +61,7 @@ import org.sleuthkit.datamodel.ContentVisitor; import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.FsContent; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData.FileKnown; @@ -107,6 +112,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile private Searcher finalSearcher; private volatile boolean searcherDone = true; //mark as done, until it's inited private Map> currentResults; + //only search images from current ingest, not images previously ingested/indexed + //accessed read-only by searcher thread + private Set curImageIds; private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy private static final Lock searcherLock = rwLock.writeLock(); private volatile int messageID = 0; @@ -158,6 +166,12 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile logger.log(Level.WARNING, "Skipping processing, module not initialized, file: " + abstractFile.getName()); return ProcessResult.OK; } + try { + //add image id of the file to the set, keeping track of images being ingested + curImageIds.add(abstractFile.getImage().getId()); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex); + } //check if we should index meta-data only when 1) it is known 2) HashDb module errored on it IngestModuleAbstractFile.ProcessResult hashDBResult = services.getAbstractFileModuleResult(hashDBModuleName); @@ -253,7 +267,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile } catch (KeywordSearchModuleException se) { logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", se); } - + //cleanup done in final searcher //postSummary(); @@ -295,6 +309,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile private void cleanup() { ingestStatus.clear(); currentResults.clear(); + curImageIds.clear(); currentSearcher = null; //finalSearcher = null; //do not collect, might be finalizing @@ -408,6 +423,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile //keeps track of all results per run not to repeat reporting the same hits currentResults = new HashMap>(); + curImageIds = new HashSet(); + indexer = new Indexer(); final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; @@ -909,6 +926,16 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile del = new TermComponentQuery(keywordQuery); } + //limit search to currently ingested images + final long imageIds[] = new long[curImageIds.size()]; + final Iterator it = curImageIds.iterator(); + for (int imageI = 0; it.hasNext(); ++imageI) { + imageIds[imageI] = it.next(); + } + //set up a filter with 1 or more image ids OR'ed + final KeywordQueryFilter imageFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.IMAGE, imageIds); + del.addFilter(imageFilter); + Map> queryResult = null; try { @@ -1084,10 +1111,10 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); } } //if has results - + //reset the status text before it goes away subProgresses[keywordsSearched].progress(""); - + ++keywordsSearched; } //for each keyword @@ -1096,10 +1123,13 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile catch (Exception ex) { logger.log(Level.WARNING, "searcher exception occurred", ex); } finally { - finalizeSearcher(); - stopWatch.stop(); - logger.log(Level.INFO, "Searcher took to run: " + stopWatch.getElapsedTimeSecs() + " secs."); - searcherLock.unlock(); + try { + finalizeSearcher(); + stopWatch.stop(); + logger.log(Level.INFO, "Searcher took to run: " + stopWatch.getElapsedTimeSecs() + " secs."); + } finally { + searcherLock.unlock(); + } } return null; @@ -1148,9 +1178,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile //this is the final searcher logger.log(Level.INFO, "The final searcher in this ingest done."); finalSearcherDone = true; - + services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchIngestModule.instance, "Completed")); - + //run module cleanup cleanup(); } else { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java index e3735b7856..a9c57dce36 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java @@ -44,9 +44,11 @@ public interface KeywordSearchQuery { /** * Set an optional filter to narrow down the search + * Adding multiple filters ANDs them together. + * For OR, add multiple ids to a single filter * @param filter filter to set on the query */ - public void setFilter(KeywordQueryFilter filter); + public void addFilter(KeywordQueryFilter filter); /** * Set an optional field to narrow down the search diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 2691533825..e62d983b1f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -49,7 +49,7 @@ public class LuceneQuery implements KeywordSearchQuery { private String queryEscaped; private boolean isEscaped; private Keyword keywordQuery = null; - private KeywordQueryFilter filter = null; + private final List filters = new ArrayList(); private String field = null; private static final int MAX_RESULTS = 20000; static final int SNIPPET_LENGTH = 50; @@ -73,8 +73,8 @@ public class LuceneQuery implements KeywordSearchQuery { } @Override - public void setFilter(KeywordQueryFilter filter) { - this.filter = filter; + public void addFilter(KeywordQueryFilter filter) { + this.filters.add(filter); } @Override @@ -204,7 +204,7 @@ public class LuceneQuery implements KeywordSearchQuery { q.setQuery(theQueryStr); q.setRows(MAX_RESULTS); q.setFields(Server.Schema.ID.toString()); - if (filter != null) { + for (KeywordQueryFilter filter : filters) { q.addFilterQuery(filter.toString()); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index 89ed96d8f8..cffa090d08 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -54,7 +54,7 @@ public class TermComponentQuery implements KeywordSearchQuery { private boolean isEscaped; private List terms; private Keyword keywordQuery = null; - private KeywordQueryFilter filter = null; + private final List filters = new ArrayList(); private String field = null; private static int MAX_TERMS_RESULTS = 20000; @@ -69,8 +69,8 @@ public class TermComponentQuery implements KeywordSearchQuery { } @Override - public void setFilter(KeywordQueryFilter filter) { - this.filter = filter; + public void addFilter(KeywordQueryFilter filter) { + this.filters.add(filter); } @Override @@ -232,8 +232,11 @@ public class TermComponentQuery implements KeywordSearchQuery { LuceneQuery filesQuery = new LuceneQuery(termStr); //filesQuery.setField(TERMS_SEARCH_FIELD); - if (filter != null) { - filesQuery.setFilter(filter); + for (KeywordQueryFilter filter : filters) { + //set filter + //note: we can't set filter query on terms query + //but setting filter query on terms results query will yield the same result + filesQuery.addFilter(filter); } try { Map> subResults = filesQuery.performQuery(); diff --git a/NEWS.txt b/NEWS.txt index 136acb9e1e..dcdde2c85f 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -9,6 +9,7 @@ Improvements: Bugfixes: +- fixed bug when keyword search ingest would search also images previously ingested, creating duplicate results - fixed crash and hang in html and excel report generation, due to special characters present