codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-10-03 15:59:52 +00:00
Code Issues Releases Wiki Activity

keyword search ingest: search only images being ingested, not other images present in the index

Browse Source
This commit is contained in:
adam-m
2013-01-09 17:31:02 -05:00
parent 2b0deb68c9
commit 44599504ed
7 changed files with 95 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
+1 -1
View File
@@ -141,7 +141,7 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
Keyword keywordQuery = new Keyword(queryStr, false);
chunksQuery = new LuceneQuery(keywordQuery);
KeywordQueryFilter contentIdFilter = new KeywordQueryFilter(FilterType.CHUNK, contentId);
chunksQuery.setFilter(contentIdFilter);
chunksQuery.addFilter(contentIdFilter);
try {
hits = chunksQuery.performQuery();
} catch (NoOpenCoreException ex) {
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java
+39 -19
View File
@@ -16,43 +16,63 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
/**
*
* Filter to select only specific id or chunks for that id
* Filter to restrict query only specific files, chunks, images
* Single filter supports multiple ids per file/chunk/image, that act as OR filter
*/
public class KeywordQueryFilter {
public static enum FilterType {FILE, CHUNK};
private long idFilter;
public static enum FilterType {
FILE, CHUNK, IMAGE
};
private long[] idFilters;
private FilterType filterType;
public KeywordQueryFilter(FilterType filterType, long id) {
this.filterType = filterType;
this.idFilter = id;
this.idFilters = new long[1];
this.idFilters[0] = id;
}
public long getIdFilter() {
return idFilter;
public KeywordQueryFilter(FilterType filterType, long[] ids) {
this.filterType = filterType;
this.idFilters = ids;
}
public long[] getIdFilters() {
return idFilters;
}
public FilterType getFilterType() {
return filterType;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(Server.Schema.ID.toString());
sb.append(":");
sb.append(Long.toString(idFilter));
if (filterType == FilterType.CHUNK) {
sb.append("_*");
String id = null;
for (int i = 0; i < idFilters.length; ++i) {
if (i > 0) {
sb.append(" "); //OR
}
long idVal = idFilters[i];
if (filterType == FilterType.IMAGE) {
id = Server.Schema.IMAGE_ID.toString();
} else {
id = Server.Schema.ID.toString();
}
sb.append(id);
sb.append(":");
sb.append(Long.toString(idVal));
if (filterType == FilterType.CHUNK) {
sb.append("_*");
}
}
return sb.toString();
}
}
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
+39 -9
View File
@@ -20,11 +20,15 @@ package org.sleuthkit.autopsy.keywordsearch;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.lang.Long;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -57,6 +61,7 @@ import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.FileKnown;
@@ -107,6 +112,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
private Searcher finalSearcher;
private volatile boolean searcherDone = true; //mark as done, until it's inited
private Map<Keyword, List<Long>> currentResults;
//only search images from current ingest, not images previously ingested/indexed
//accessed read-only by searcher thread
private Set<Long> curImageIds;
private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy
private static final Lock searcherLock = rwLock.writeLock();
private volatile int messageID = 0;
@@ -158,6 +166,12 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
logger.log(Level.WARNING, "Skipping processing, module not initialized, file: " + abstractFile.getName());
return ProcessResult.OK;
}
try {
//add image id of the file to the set, keeping track of images being ingested
curImageIds.add(abstractFile.getImage().getId());
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex);
}
//check if we should index meta-data only when 1) it is known 2) HashDb module errored on it
IngestModuleAbstractFile.ProcessResult hashDBResult = services.getAbstractFileModuleResult(hashDBModuleName);
@@ -253,7 +267,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
} catch (KeywordSearchModuleException se) {
logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", se);
}
//cleanup done in final searcher
//postSummary();
@@ -295,6 +309,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
private void cleanup() {
ingestStatus.clear();
currentResults.clear();
curImageIds.clear();
currentSearcher = null;
//finalSearcher = null; //do not collect, might be finalizing
@@ -408,6 +423,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
//keeps track of all results per run not to repeat reporting the same hits
currentResults = new HashMap<Keyword, List<Long>>();
curImageIds = new HashSet<Long>();
indexer = new Indexer();
final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000;
@@ -909,6 +926,16 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
del = new TermComponentQuery(keywordQuery);
}
//limit search to currently ingested images
final long imageIds[] = new long[curImageIds.size()];
final Iterator<Long> it = curImageIds.iterator();
for (int imageI = 0; it.hasNext(); ++imageI) {
imageIds[imageI] = it.next();
}
//set up a filter with 1 or more image ids OR'ed
final KeywordQueryFilter imageFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.IMAGE, imageIds);
del.addFilter(imageFilter);
Map<String, List<ContentHit>> queryResult = null;
try {
@@ -1084,10 +1111,10 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts));
}
} //if has results
//reset the status text before it goes away
subProgresses[keywordsSearched].progress("");
++keywordsSearched;
} //for each keyword
@@ -1096,10 +1123,13 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
catch (Exception ex) {
logger.log(Level.WARNING, "searcher exception occurred", ex);
} finally {
finalizeSearcher();
stopWatch.stop();
logger.log(Level.INFO, "Searcher took to run: " + stopWatch.getElapsedTimeSecs() + " secs.");
searcherLock.unlock();
try {
finalizeSearcher();
stopWatch.stop();
logger.log(Level.INFO, "Searcher took to run: " + stopWatch.getElapsedTimeSecs() + " secs.");
} finally {
searcherLock.unlock();
}
}
return null;
@@ -1148,9 +1178,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
//this is the final searcher
logger.log(Level.INFO, "The final searcher in this ingest done.");
finalSearcherDone = true;
services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchIngestModule.instance, "Completed"));
//run module cleanup
cleanup();
} else {
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java
+3 -1
View File
@@ -44,9 +44,11 @@ public interface KeywordSearchQuery {
/**
* Set an optional filter to narrow down the search
* Adding multiple filters ANDs them together.
* For OR, add multiple ids to a single filter
* @param filter filter to set on the query
*/
public void setFilter(KeywordQueryFilter filter);
public void addFilter(KeywordQueryFilter filter);
/**
* Set an optional field to narrow down the search
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
+4 -4
View File
@@ -49,7 +49,7 @@ public class LuceneQuery implements KeywordSearchQuery {
private String queryEscaped;
private boolean isEscaped;
private Keyword keywordQuery = null;
private KeywordQueryFilter filter = null;
private final List <KeywordQueryFilter> filters = new ArrayList<KeywordQueryFilter>();
private String field = null;
private static final int MAX_RESULTS = 20000;
static final int SNIPPET_LENGTH = 50;
@@ -73,8 +73,8 @@ public class LuceneQuery implements KeywordSearchQuery {
}
@Override
public void setFilter(KeywordQueryFilter filter) {
this.filter = filter;
public void addFilter(KeywordQueryFilter filter) {
this.filters.add(filter);
}
@Override
@@ -204,7 +204,7 @@ public class LuceneQuery implements KeywordSearchQuery {
q.setQuery(theQueryStr);
q.setRows(MAX_RESULTS);
q.setFields(Server.Schema.ID.toString());
if (filter != null) {
for (KeywordQueryFilter filter : filters) {
q.addFilterQuery(filter.toString());
}
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
+8 -5
View File
@@ -54,7 +54,7 @@ public class TermComponentQuery implements KeywordSearchQuery {
private boolean isEscaped;
private List<Term> terms;
private Keyword keywordQuery = null;
private KeywordQueryFilter filter = null;
private final List<KeywordQueryFilter> filters = new ArrayList<KeywordQueryFilter>();
private String field = null;
private static int MAX_TERMS_RESULTS = 20000;
@@ -69,8 +69,8 @@ public class TermComponentQuery implements KeywordSearchQuery {
}
@Override
public void setFilter(KeywordQueryFilter filter) {
this.filter = filter;
public void addFilter(KeywordQueryFilter filter) {
this.filters.add(filter);
}
@Override
@@ -232,8 +232,11 @@ public class TermComponentQuery implements KeywordSearchQuery {
LuceneQuery filesQuery = new LuceneQuery(termStr);
//filesQuery.setField(TERMS_SEARCH_FIELD);
if (filter != null) {
filesQuery.setFilter(filter);
for (KeywordQueryFilter filter : filters) {
//set filter
//note: we can't set filter query on terms query
//but setting filter query on terms results query will yield the same result
filesQuery.addFilter(filter);
}
try {
Map<String, List<ContentHit>> subResults = filesQuery.performQuery();
NEWS.txt
+1
View File
@@ -9,6 +9,7 @@ Improvements:
Bugfixes:
- fixed bug when keyword search ingest would search also images previously ingested, creating duplicate results
- fixed crash and hang in html and excel report generation, due to special characters present