codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-09-06 02:24:30 +00:00
Code Issues Releases Wiki Activity

Merge branch 'master' of https://github.com/sleuthkit/autopsy

Browse Source
This commit is contained in:
Tim McIver
2013-01-02 17:42:50 -05:00
parent 2081639d30 d10cf24a1a
commit 6285beaa5b
11 changed files with 222 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Core/src/org/sleuthkit/autopsy/coreutils/StringExtract.java
View File

@@ -52,6 +52,12 @@ public class StringExtract {
private List<SCRIPT> enabledScripts;
private boolean enableUTF8;
private boolean enableUTF16;
//stored and reused results
private final StringExtractResult resUTF16En1 = new StringExtractResult();
private final StringExtractResult resUTF16En2 = new StringExtractResult();
private final StringExtractResult resUTF8 = new StringExtractResult();
/**
* supported scripts, can be overridden with enableScriptX methods
*/
@@ -202,22 +208,21 @@ public class StringExtract {
//extract using all methods and see which one wins
StringExtractResult resUTF16 = null;
boolean runUTF16 = false;
if (enableUTF16 && curOffset % 2 == 0) {
StringExtractResult resUTF16En1 = extractUTF16(buff, len, curOffset, true);
StringExtractResult resUTF16En2 = extractUTF16(buff, len, curOffset, false);
runUTF16 = true;
extractUTF16(buff, len, curOffset, true, resUTF16En1);
extractUTF16(buff, len, curOffset, false, resUTF16En2);
resUTF16 = resUTF16En1.numChars > resUTF16En2.numChars ? resUTF16En1 : resUTF16En2;
}
//results.add(extractUTF8(buff, len, curOffset));
StringExtractResult resUTF8 = null;
if (enableUTF8) {
resUTF8 = extractUTF8(buff, len, curOffset);
extractUTF8(buff, len, curOffset, resUTF8);
}
StringExtractResult resWin = null;
if (enableUTF8 && enableUTF16) {
resWin = resUTF16 != null && resUTF16.numChars > resUTF8.numChars ? resUTF16 : resUTF8;
resWin = runUTF16 && resUTF16.numChars > resUTF8.numChars ? resUTF16 : resUTF8;
} else if (enableUTF16){
resWin = resUTF16;
}
@@ -250,7 +255,7 @@ public class StringExtract {
}
}
//build up the result
//build up the final result
StringExtractResult res = new StringExtractResult();
res.numBytes = processedBytes;
res.numChars = curStringLen;
@@ -261,9 +266,9 @@ public class StringExtract {
return res;
}
private StringExtractResult extractUTF16(byte[] buff, int len, int offset, boolean endianSwap) {
StringExtractResult res = new StringExtractResult();
private StringExtractResult extractUTF16(byte[] buff, int len, int offset, boolean endianSwap, final StringExtractResult res) {
res.reset();
int curOffset = offset;
final StringBuilder tempString = new StringBuilder();
@@ -352,8 +357,8 @@ public class StringExtract {
return res;
}
private StringExtractResult extractUTF8(byte[] buff, int len, int offset) {
StringExtractResult res = new StringExtractResult();
private StringExtractResult extractUTF8(byte[] buff, int len, int offset, final StringExtractResult res) {
res.reset();
int curOffset = offset;
int ch = 0; //character being extracted
@@ -608,6 +613,15 @@ public class StringExtract {
int firstUnprocessedOff; ///< first byte past the last byte used in extraction, offset+numBytes for a single result, but we keep track of it for multiple extractions
String textString; ///< the actual text string extracted, of numChars long
void reset() {
offset = 0;
numBytes = 0;
numChars = 0;
firstUnprocessedOff = 0;
textString = null;
}
public int getFirstUnprocessedOff() {
return firstUnprocessedOff;
}

2
KeywordSearch/ivy.xml
View File

@@ -20,6 +20,6 @@
<dependency conf="start-solr->default" org="org.mortbay.jetty" name="start" rev="6.1.26"/>
<dependency conf="jetty-libs->default" org="org.mortbay.jetty" name="jetty" rev="6.1.26"/>
<dependency conf="jetty-libs->default" org="org.mortbay.jetty" name="jsp-2.1" rev="6.1.14"/>
<dependency conf="autopsy->*" org="net.htmlparser.jericho" name="jericho-html" rev="3.2"/>
<dependency conf="autopsy->*" org="net.htmlparser.jericho" name="jericho-html" rev="3.3"/>
</dependencies>
</ivy-module>

24
KeywordSearch/nbproject/project.xml
View File

@@ -115,6 +115,10 @@
<runtime-relative-path>ext/commons-lang-2.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-lang-2.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/log4j-1.2.17.jar</runtime-relative-path>
<binary-origin>release/modules/ext/log4j-1.2.17.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jcl-over-slf4j-1.6.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jcl-over-slf4j-1.6.4.jar</binary-origin>
@@ -215,14 +219,14 @@
<runtime-relative-path>ext/boilerpipe-1.1.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/boilerpipe-1.1.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jericho-html-3.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jericho-html-3.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/org.osgi.compendium-4.0.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/org.osgi.compendium-4.0.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/slf4j-api-1.7.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/slf4j-api-1.7.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jempbox-1.7.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jempbox-1.7.0.jar</binary-origin>
@@ -279,14 +283,14 @@
<runtime-relative-path>ext/poi-scratchpad-3.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/poi-scratchpad-3.8.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/log4j-1.2.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/log4j-1.2.16.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/poi-ooxml-schemas-3.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/poi-ooxml-schemas-3.8.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jericho-html-3.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jericho-html-3.3.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/bcprov-jdk15-1.45.jar</runtime-relative-path>
<binary-origin>release/modules/ext/bcprov-jdk15-1.45.jar</binary-origin>
@@ -299,10 +303,6 @@
<runtime-relative-path>ext/apache-mime4j-dom-0.7.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/apache-mime4j-dom-0.7.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/slf4j-api-1.6.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/slf4j-api-1.6.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/geronimo-stax-api_1.0_spec-1.0.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/geronimo-stax-api_1.0_spec-1.0.1.jar</binary-origin>

10
KeywordSearch/release/solr/solr/conf/schema.xml
View File

@@ -209,7 +209,7 @@
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="10000"/>
<filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="100000"/>
</analyzer>
</fieldType>
@@ -222,7 +222,7 @@
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- workaround to bug in Solr 4.0 to set LimitTokenCountFilterFactory maxTokenCount -->
<filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="10000"/>
<filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="100000"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<!-- in this example, we will only use synonyms at query time
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
@@ -232,7 +232,7 @@
<analyzer type="query">
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- workaround to bug in Solr 4.0 to set LimitTokenCountFilterFactory maxTokenCount -->
<filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="10000"/>
<filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="100000"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.LowerCaseFilterFactory"/>
@@ -504,7 +504,7 @@
<field name="id" type="string" indexed="true" stored="true" required="true" />
<!-- The content field holds the text extracted by SolrCell -->
<field name="content" type="text_general" indexed="true" stored="true"/>
<field name="content" type="text_general" indexed="true" stored="true" termVectors="true" termPositions="true" termOffsets="true" />
<!-- The strings field holds strings extracted from files that SolrCell doesn't support -->
<field name="strings" type="text_general" indexed="true" stored="true"/>
<field name="file_name" type="text_general" indexed="true" stored="true"/>
@@ -545,7 +545,7 @@
<!-- field with white-space tokenized words for TermsComponent regex search (useful for fast search of IP addresses, URLs, certain phone numbers)
also be useful for Lucene based queries containing special characters-->
<field name="content_ws" type="text_ws" indexed="true" stored="false" multiValued="true"/>
<field name="content_ws" type="text_ws" indexed="true" stored="false" />
<!-- Uncommenting the following will create a "timestamp" field using
a default value of "NOW" to indicate when each document was indexed.

10
KeywordSearch/release/solr/solr/conf/solrconfig.xml
View File

@@ -903,14 +903,16 @@
<!-- Highlighting defaults -->
<str name="hl">on</str>
<str name="hl.fl">content features title name</str>
<str name="hl.fl">text content file_name</str>
<str name="hl.encoder">html</str>
<str name="f.name.hl.fragsize">0</str>
<str name="f.name.hl.alternateField">file_name</str>
<str name="hl.simple.pre">&lt;b&gt;</str>
<str name="hl.simple.post">&lt;/b&gt;</str>
<str name="f.title.hl.fragsize">0</str>
<str name="f.title.hl.alternateField">title</str>
<str name="f.name.hl.fragsize">0</str>
<str name="f.name.hl.alternateField">name</str>
<str name="f.content.hl.snippets">3</str>
<str name="f.content.hl.fragsize">200</str>
<str name="f.content.hl.alternateField">content</str>
@@ -1546,10 +1548,12 @@
<!-- Configure the standard encoder -->
<encoder name="html"
default="true"
class="solr.highlight.HtmlEncoder" />
<!-- Configure the standard fragListBuilder -->
<fragListBuilder name="simple"
default="true"
class="solr.highlight.SimpleFragListBuilder"/>
<!-- Configure the single fragListBuilder -->

68
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
View File

@@ -31,6 +31,7 @@ import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.autopsy.datamodel.HighlightLookup;
import org.sleuthkit.autopsy.keywordsearch.KeywordQueryFilter.FilterType;
import org.sleuthkit.datamodel.Content;
@@ -62,6 +63,7 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
private Map<String, List<ContentHit>> hits = null; //original hits that may get passed in
private String originalQuery = null; //or original query if hits are not available
private boolean inited = false;
private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
HighlightedMatchesSource(Content content, String keywordHitQuery, boolean isRegex) {
this.content = content;
@@ -119,21 +121,17 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
if (hasChunks) {
//extract pages of interest, sorted
final long contentId = content.getId();
if (hits == null) {
//special case, aka in case of dir tree, we don't know which chunks
//reperform search query for the content to get matching chunks info
KeywordSearchQuery chunksQuery = null;
/**
Keyword keywordQuery = new Keyword(this.originalQuery, !isRegex);
if (this.isRegex)
{
chunksQuery = new TermComponentQuery(keywordQuery);
} else {
chunksQuery = new LuceneQuery(keywordQuery);
chunksQuery.escape();
}
* Keyword keywordQuery = new Keyword(this.originalQuery,
* !isRegex); if (this.isRegex) { chunksQuery = new
* TermComponentQuery(keywordQuery); } else { chunksQuery = new
* LuceneQuery(keywordQuery); chunksQuery.escape(); }
*/
String queryStr = KeywordSearchUtil.escapeLuceneQuery(this.keywordHitQuery);
if (isRegex) {
@@ -164,9 +162,11 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
}
//set page to first page having highlights
if (pagesSorted.isEmpty())
if (pagesSorted.isEmpty()) {
this.currentPage = 0;
else this.currentPage = pagesSorted.first();
} else {
this.currentPage = pagesSorted.first();
}
for (Integer page : pagesSorted) {
hitsPages.put(page, 0); //unknown number of matches in the page
@@ -240,15 +240,17 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
@Override
public boolean hasNextItem() {
if (!this.pagesToHits.containsKey(currentPage))
if (!this.pagesToHits.containsKey(currentPage)) {
return false;
}
return this.pagesToHits.get(currentPage) < this.hitsPages.get(currentPage);
}
@Override
public boolean hasPreviousItem() {
if (!this.pagesToHits.containsKey(currentPage))
if (!this.pagesToHits.containsKey(currentPage)) {
return false;
}
return this.pagesToHits.get(currentPage) > 1;
}
@@ -274,8 +276,9 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
@Override
public int currentItem() {
if (!this.pagesToHits.containsKey(currentPage))
if (!this.pagesToHits.containsKey(currentPage)) {
return 0;
}
return pagesToHits.get(currentPage);
}
@@ -307,6 +310,9 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
}
SolrQuery q = new SolrQuery();
q.setShowDebugInfo(DEBUG); //debug
String queryStr = null;
if (isRegex) {
StringBuilder sb = new StringBuilder();
@@ -318,13 +324,15 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
if (group) {
sb.append("\"");
}
q.setQuery(sb.toString());
queryStr = sb.toString();
} else {
//use default field, simplifies query
//always force grouping/quotes
q.setQuery(KeywordSearchUtil.quoteQuery(highlightQuery));
queryStr = KeywordSearchUtil.quoteQuery(highlightQuery);
}
q.setQuery(queryStr);
final long contentId = content.getId();
String contentIdStr = Long.toString(contentId);
@@ -332,13 +340,26 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
contentIdStr += "_" + Integer.toString(this.currentPage);
}
final String filterQuery = Server.Schema.ID.toString() + ":" + contentIdStr;
q.addFilterQuery(filterQuery);
q.addHighlightField(highLightField); //for exact highlighting, try content_ws field (with stored="true" in Solr schema)
q.setHighlightSimplePre(HIGHLIGHT_PRE);
q.setHighlightSimplePost(HIGHLIGHT_POST);
q.setHighlightFragsize(0); // don't fragment the highlight
q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //analyze all content
//need to use original highlighter (as opposed to snippets), because FVH does not seem to support fragsize=0 to get entire content
//https://issues.apache.org/jira/browse/SOLR-1268?attachmentSortBy=dateTime
q.setHighlightSimplePre(HIGHLIGHT_PRE); //original highlighter only
q.setHighlightSimplePost(HIGHLIGHT_POST); //original highlighter only
q.setHighlightFragsize(0); // don't fragment the highlight, works with original highlighter only
//tune the highlighter
//q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one
//q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only
//q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only
//q.setParam("hl.fragListBuilder", "simple"); //makes sense for FastVectorHighlighter only
//docs says makes sense for the original Highlighter only, but not really
q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED);
try {
QueryResponse response = solrServer.query(q, METHOD.POST);
@@ -357,7 +378,7 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
highlightedContent = insertAnchors(highlightedContent);
return "<pre>" + highlightedContent + "</pre>";
return "<html><pre>" + highlightedContent + "</pre></html>";
}
} catch (NoOpenCoreException ex) {
logger.log(Level.WARNING, "Couldn't query markup for page: " + currentPage, ex);
@@ -385,8 +406,9 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
@Override
public int getNumberHits() {
if (!this.hitsPages.containsKey(this.currentPage))
if (!this.hitsPages.containsKey(this.currentPage)) {
return 0;
}
return this.hitsPages.get(this.currentPage);
}

120
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@@ -86,10 +86,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
return time;
}
};
private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName());
public static final String MODULE_NAME = "Keyword Search";
public static final String MODULE_DESCRIPTION = "Performs file indexing and periodic search using keywords and regular expressions in lists.";
public static final String MODULE_DESCRIPTION = "Performs file indexing and periodic search using keywords and regular expressions in lists.";
final public static String MODULE_VERSION = "1.0";
private String args;
private static KeywordSearchIngestModule instance = null;
@@ -116,10 +115,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
private SleuthkitCase caseHandle = null;
private static List<AbstractFileExtract> textExtractors;
private static AbstractFileStringExtract stringExtractor;
private boolean initialized = false;
private final GetIsFileKnownV getIsFileKnown = new GetIsFileKnownV();
private KeywordSearchConfigurationPanel panel;
@@ -131,7 +127,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
//private constructor to ensure singleton instance
private KeywordSearchIngestModule() {
}
/**
@@ -159,6 +154,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
if (initialized == false) //error initializing indexing/Solr
{
logger.log(Level.WARNING, "Skipping processing, module not initialized, file: " + abstractFile.getName());
return ProcessResult.OK;
}
@@ -170,12 +166,11 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
indexer.indexFile(abstractFile, false);
//notify depending module that keyword search (would) encountered error for this file
return ProcessResult.ERROR;
}
else if (KeywordSearchSettings.getSkipKnown() && abstractFile.accept(getIsFileKnown) == true) {
} else if (KeywordSearchSettings.getSkipKnown() && abstractFile.accept(getIsFileKnown) == true) {
//index meta-data only
indexer.indexFile(abstractFile, false);
return ProcessResult.OK;
}
}
if (processedFiles == false) {
processedFiles = true;
@@ -186,13 +181,14 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
//index the file and content (if the content is supported)
indexer.indexFile(abstractFile, true);
return ProcessResult.OK;
}
/**
* Process content hierarchy and return true if content is a file and is set as known
* Process content hierarchy and return true if content is a file and is set
* as known
*/
private class GetIsFileKnownV extends ContentVisitor.Default<Boolean> {
@@ -200,12 +196,11 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
protected Boolean defaultVisit(Content cntnt) {
return false;
}
@Override
public Boolean visit(File file) {
return file.getKnown() == FileKnown.KNOWN;
}
}
/**
@@ -246,7 +241,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
}
//log number of files / chunks in index
//signal a potential change in number of indexed files
//signal a potential change in number of indexed files
try {
final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles();
final int numIndexedChunks = KeywordSearch.getServer().queryNumIndexedChunks();
@@ -256,9 +251,10 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex);
} catch (KeywordSearchModuleException se) {
logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", se);
} finally {
cleanup();
}
//postSummary();
}
@@ -288,6 +284,33 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
commit();
//postSummary();
cleanup();
}
/**
* Common cleanup code when module stops or completes
*/
private void cleanup() {
ingestStatus.clear();
currentResults.clear();
currentSearcher = null;
finalSearcher = null;
commitTimer.stop();
searchTimer.stop();
commitTimer = null;
searchTimer = null;
textExtractors.clear();
textExtractors = null;
stringExtractor = null;
keywords.clear();
keywordLists.clear();
keywordToList.clear();
initialized = false;
}
@Override
@@ -299,7 +322,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
public String getDescription() {
return MODULE_DESCRIPTION;
}
@Override
public String getVersion() {
return MODULE_VERSION;
@@ -314,7 +337,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
public void setArguments(String args) {
this.args = args;
}
/**
* Initializes the module for new ingest run Sets up threads, timers,
@@ -326,20 +348,20 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
logger.log(Level.INFO, "init()");
services = IngestServices.getDefault();
initialized = false;
caseHandle = Case.getCurrentCase().getSleuthkitCase();
ingester = Server.getIngester();
final Server server = KeywordSearch.getServer();
try {
if (! server.isRunning()) {
if (!server.isRunning()) {
String msg = "Keyword search server was not properly initialized, cannot run keyword search ingest. ";
logger.log(Level.SEVERE, msg);
String details = msg + "Please try restarting the OS and the application";
services.postMessage(IngestMessage.createErrorMessage(++messageID, instance, msg, details));
return;
}
} catch (KeywordSearchModuleException ex) {
logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex);
@@ -352,8 +374,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
stringExtractor = new AbstractFileStringExtract();
stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts());
stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions());
//log the scripts used for debugging
final StringBuilder sbScripts = new StringBuilder();
for (SCRIPT s : KeywordSearchSettings.getStringExtractScripts()) {
@@ -376,7 +398,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
initKeywords();
if (keywords.isEmpty() || keywordLists.isEmpty()) {
services.postMessage(IngestMessage.createWarningMessage(++messageID, instance, "No keywords in keyword list.", "Only indexing will be done and and keyword search will be skipped (you can still add keyword lists using the Keyword Lists - Add to Ingest)."));
services.postMessage(IngestMessage.createWarningMessage(++messageID, instance, "No keywords in keyword list.", "Only indexing will be done and and keyword search will be skipped (you can still add keyword lists using the Keyword Lists - Add to Ingest)."));
}
processedFiles = false;
@@ -531,23 +553,24 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
}
/**
* Initialize the keyword search lists and associated keywords from the XML loader
* Use the lists to ingest that are set in the permanent XML configuration
* Initialize the keyword search lists and associated keywords from the XML
* loader Use the lists to ingest that are set in the permanent XML
* configuration
*/
private void initKeywords() {
addKeywordLists(null);
}
/**
* If ingest is ongoing, this will add
* additional keyword search lists to the ongoing ingest
* The lists to add may be temporary and not necessary set to be
* added to ingest permanently in the XML configuration.
* The lists will be reset back to original (permanent configuration state)
* on the next ingest.
* If ingest is ongoing, this will add additional keyword search lists to
* the ongoing ingest The lists to add may be temporary and not necessary
* set to be added to ingest permanently in the XML configuration. The lists
* will be reset back to original (permanent configuration state) on the
* next ingest.
*
* @param listsToAdd lists to add temporarily to the ongoing ingest
*/
void addKeywordLists(List<String>listsToAdd) {
void addKeywordLists(List<String> listsToAdd) {
KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent();
keywords.clear();
@@ -555,11 +578,11 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
keywordToList.clear();
StringBuilder sb = new StringBuilder();
for (KeywordSearchList list : loader.getListsL()) {
final String listName = list.getName();
if (list.getUseForIngest() == true
|| (listsToAdd != null && listsToAdd.contains(listName)) ) {
|| (listsToAdd != null && listsToAdd.contains(listName))) {
keywordLists.add(listName);
sb.append(listName).append(" ");
}
@@ -571,9 +594,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
}
}
logger.log(Level.INFO, "Set new effective keyword lists: " + sb.toString());
}
List<String> getKeywordLists() {
@@ -696,8 +719,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.VIRTUAL_DIR)) {
//skip indexing of virtual dirs (no content, no real name) - will index children files
return;
}
else if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
} else if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
fsContent = (FsContent) aFile;
}
@@ -791,7 +813,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
private List<Keyword> keywords; //keywords to search
private List<String> keywordLists; // lists currently being searched
private Map<String, KeywordSearchList> keywordToList; //keyword to list name mapping
private ProgressHandle progress;
private final Logger logger = Logger.getLogger(Searcher.class.getName());
private boolean finalRun = false;
@@ -811,9 +832,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
@Override
protected Object doInBackground() throws Exception {
if (finalRun) {
logger.log(Level.INFO, "Pending start of new (final) searcher");
}
else {
logger.log(Level.INFO, "Pending start of new (final) searcher");
} else {
logger.log(Level.INFO, "Pending start of new searcher");
}
@@ -906,8 +926,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
if (this.isCancelled()) {
logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: " + keywordQuery.getQuery());
return null;
}
}
List<ContentHit> contentHitsAll = newResults.get(hitTerm);
Map<AbstractFile, Integer> contentHitsFlattened = ContentHit.flattenResults(contentHitsAll);
for (final AbstractFile hitFile : contentHitsFlattened.keySet()) {
@@ -1079,8 +1099,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
//this is the final searcher
logger.log(Level.INFO, "The final searcher in this ingest done.");
finalSearcherDone = true;
//reset current resuls earlier to potentially garbage collect sooner
currentResults = new HashMap<Keyword, List<Long>>();
services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchIngestModule.instance, "Completed"));
} else {

46
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
View File

@@ -33,6 +33,7 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
@@ -50,11 +51,15 @@ public class LuceneQuery implements KeywordSearchQuery {
private Keyword keywordQuery = null;
private KeywordQueryFilter filter = null;
private String field = null;
//use different highlight Solr fields for regex and literal search
private static final int MAX_RESULTS = 10000;
static final int SNIPPET_LENGTH = 50;
//can use different highlight schema fields for regex and literal search
static final String HIGHLIGHT_FIELD_LITERAL = Server.Schema.CONTENT.toString();
static final String HIGHLIGHT_FIELD_REGEX = Server.Schema.CONTENT.toString();
//TODO use content_ws stored="true" in solr schema for perfect highlight hits
//static final String HIGHLIGHT_FIELD_REGEX = Server.Schema.CONTENT_WS.toString()
private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
public LuceneQuery(Keyword keywordQuery) {
this(keywordQuery.getQuery());
@@ -180,11 +185,11 @@ public class LuceneQuery implements KeywordSearchQuery {
List<ContentHit> matches = new ArrayList<ContentHit>();
boolean allMatchesFetched = false;
final int ROWS_PER_FETCH = 10000;
final Server solrServer = KeywordSearch.getServer();
SolrQuery q = new SolrQuery();
q.setShowDebugInfo(DEBUG); //debug
//set query, force quotes/grouping around all literal queries
final String groupedQuery = KeywordSearchUtil.quoteQuery(queryEscaped);
@@ -197,20 +202,20 @@ public class LuceneQuery implements KeywordSearchQuery {
}
q.setQuery(theQueryStr);
q.setRows(ROWS_PER_FETCH);
q.setRows(MAX_RESULTS);
q.setFields(Server.Schema.ID.toString());
if (filter != null) {
q.addFilterQuery(filter.toString());
}
for (int start = 0; !allMatchesFetched; start = start + ROWS_PER_FETCH) {
for (int start = 0; !allMatchesFetched; start = start + MAX_RESULTS) {
q.setStart(start);
try {
QueryResponse response = solrServer.query(q, METHOD.POST);
SolrDocumentList resultList = response.getResults();
long results = resultList.getNumFound();
allMatchesFetched = start + ROWS_PER_FETCH >= results;
allMatchesFetched = start + MAX_RESULTS >= results;
SleuthkitCase sc = null;
try {
sc = Case.getCurrentCase().getSleuthkitCase();
@@ -289,8 +294,6 @@ public class LuceneQuery implements KeywordSearchQuery {
* @return
*/
public static String querySnippet(String query, long contentID, int chunkID, boolean isRegex, boolean group) throws NoOpenCoreException {
final int SNIPPET_LENGTH = 45;
Server solrServer = KeywordSearch.getServer();
String highlightField = null;
@@ -302,6 +305,8 @@ public class LuceneQuery implements KeywordSearchQuery {
SolrQuery q = new SolrQuery();
String queryStr = null;
if (isRegex) {
StringBuilder sb = new StringBuilder();
sb.append(highlightField).append(":");
@@ -313,12 +318,14 @@ public class LuceneQuery implements KeywordSearchQuery {
sb.append("\"");
}
q.setQuery(sb.toString());
queryStr = sb.toString();
} else {
//simplify query/escaping and use default field
//always force grouping/quotes
q.setQuery(KeywordSearchUtil.quoteQuery(query));
queryStr = KeywordSearchUtil.quoteQuery(query);
}
q.setQuery(queryStr);
String contentIDStr = null;
@@ -329,13 +336,28 @@ public class LuceneQuery implements KeywordSearchQuery {
}
String idQuery = Server.Schema.ID.toString() + ":" + contentIDStr;
q.setShowDebugInfo(DEBUG); //debug
q.addFilterQuery(idQuery);
q.addHighlightField(highlightField);
q.setHighlightSimplePre("&laquo;");
q.setHighlightSimplePost("&raquo;");
//q.setHighlightSimplePre("&laquo;"); //original highlighter only
//q.setHighlightSimplePost("&raquo;"); //original highlighter only
q.setHighlightSnippets(1);
q.setHighlightFragsize(SNIPPET_LENGTH);
q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //analyze all content SLOW! consider lowering
//tune the highlighter
q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one
q.setParam("hl.tag.pre", "&laquo;"); //makes sense for FastVectorHighlighter only
q.setParam("hl.tag.post", "&laquo;"); //makes sense for FastVectorHighlighter only
q.setParam("hl.fragListBuilder", "simple"); //makes sense for FastVectorHighlighter only
//Solr bug if fragCharSize is smaller than Query string, StringIndexOutOfBoundsException is thrown.
q.setParam("hl.fragCharSize", Integer.toString(queryStr.length())); //makes sense for FastVectorHighlighter only
//docs says makes sense for the original Highlighter only, but not really
//analyze all content SLOW! consider lowering
q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED);
try {
QueryResponse response = solrServer.query(q);

15
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@@ -118,7 +118,7 @@ public class Server {
}
},
};
public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but option may not be supported.
public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
//max content size we can send to Solr
public static final long MAX_CONTENT_SIZE = 1L * 1024 * 1024 * 1024;
private static final Logger logger = Logger.getLogger(Server.class.getName());
@@ -139,6 +139,7 @@ public class Server {
static final int DEFAULT_SOLR_STOP_PORT = 34343;
private int currentSolrServerPort = 0;
private int currentSolrStopPort = 0;
private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
public enum CORE_EVT_STATES {
@@ -257,7 +258,6 @@ public class Server {
public void run() {
InputStreamReader isr = new InputStreamReader(stream);
BufferedReader br = new BufferedReader(isr);
final Version.Type builtType = Version.getBuildType();
try {
OutputStreamWriter osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset());
BufferedWriter bw = new BufferedWriter(osw);
@@ -265,7 +265,7 @@ public class Server {
while (doRun && (line = br.readLine()) != null) {
bw.write(line);
bw.newLine();
if (builtType == Version.Type.DEVELOPMENT) {
if (DEBUG) {
//flush buffers if dev version for debugging
bw.flush();
}
@@ -292,7 +292,7 @@ public class Server {
String loggingPropertiesOpt = " -Djava.util.logging.config.file=";
String loggingPropertiesFilePath = instanceDir + File.separator + "conf" + File.separator;
if (Version.getBuildType().equals(Version.Type.DEVELOPMENT)) {
if (DEBUG) {
loggingPropertiesFilePath += "logging-development.properties";
} else {
loggingPropertiesFilePath += "logging-release.properties";
@@ -772,13 +772,6 @@ public class Server {
this.name = name;
this.solrCore = new HttpSolrServer(solrUrl + "/" + name);
try {
solrCore.optimize(true, true);
} catch (SolrServerException ex) {
logger.log(Level.SEVERE, "Error setting optimize on solr core", ex);
} catch (IOException ex) {
logger.log(Level.SEVERE, "Error setting optimize on solr core", ex);
}
//TODO test these settings
//solrCore.setSoTimeout(1000 * 60); // socket read timeout, make large enough so can index larger files

16
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
View File

@@ -33,6 +33,7 @@ import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.TermsResponse;
import org.apache.solr.client.solrj.response.TermsResponse.Term;
import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
@@ -55,6 +56,9 @@ public class TermComponentQuery implements KeywordSearchQuery {
private Keyword keywordQuery = null;
private KeywordQueryFilter filter = null;
private String field = null;
private static int MAX_TERMS_RESULTS = 20000;
private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
public TermComponentQuery(Keyword keywordQuery) {
this.keywordQuery = keywordQuery;
@@ -217,9 +221,12 @@ public class TermComponentQuery implements KeywordSearchQuery {
Map<String, List<ContentHit>> results = new HashMap<String, List<ContentHit>>();
final SolrQuery q = createQuery();
q.setShowDebugInfo(DEBUG);
q.setTermsLimit(MAX_TERMS_RESULTS);
terms = executeQuery(q);
int resultSize = 0;
for (Term term : terms) {
final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
@@ -232,7 +239,9 @@ public class TermComponentQuery implements KeywordSearchQuery {
Map<String, List<ContentHit>> subResults = filesQuery.performQuery();
Set<ContentHit> filesResults = new HashSet<ContentHit>();
for (String key : subResults.keySet()) {
filesResults.addAll(subResults.get(key));
List<ContentHit> keyRes = subResults.get(key);
resultSize += keyRes.size();
filesResults.addAll(keyRes);
}
results.put(term.getTerm(), new ArrayList<ContentHit>(filesResults));
} catch (NoOpenCoreException e) {
@@ -243,6 +252,9 @@ public class TermComponentQuery implements KeywordSearchQuery {
}
}
//TODO limit how many results we store, not to hit memory limits
logger.log(Level.INFO, "Regex # results: " + resultSize);
return results;

5
NEWS.txt
View File

@@ -1,14 +1,17 @@
---------------- VERSION Current (dev) --------------
New features:
- Documented report module API
Improvements:
- Keyword search indexing and search speed improvements
- Remake of reporting UI and functionality
- Significant increase in reporting speed
- Documented report module API
- Upgrade to Solr4.0 / Tika 1.2
Bugfixes:
- Keyword search will index and search entire extracted content from files
---------------- VERSION 3.0.2 --------------