From d660df294004a27d1383010090689bc39713c235 Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Fri, 28 Dec 2012 13:32:15 -0500
Subject: [PATCH 1/8] upgrade jericho parser from 3.2 to 3.3

---
 KeywordSearch/ivy.xml               |  2 +-
 KeywordSearch/nbproject/project.xml | 24 ++++++++++++------------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/KeywordSearch/ivy.xml b/KeywordSearch/ivy.xml
index 800f7302ac..3dac76b7db 100644
--- a/KeywordSearch/ivy.xml
+++ b/KeywordSearch/ivy.xml
@@ -20,6 +20,6 @@
         <dependency conf="start-solr->default" org="org.mortbay.jetty" name="start" rev="6.1.26"/>
         <dependency conf="jetty-libs->default" org="org.mortbay.jetty" name="jetty" rev="6.1.26"/>
         <dependency conf="jetty-libs->default" org="org.mortbay.jetty" name="jsp-2.1" rev="6.1.14"/>
-        <dependency conf="autopsy->*" org="net.htmlparser.jericho" name="jericho-html" rev="3.2"/>
+        <dependency conf="autopsy->*" org="net.htmlparser.jericho" name="jericho-html" rev="3.3"/>
     </dependencies>
 </ivy-module>
diff --git a/KeywordSearch/nbproject/project.xml b/KeywordSearch/nbproject/project.xml
index 7aaa857d10..36b2e796e9 100644
--- a/KeywordSearch/nbproject/project.xml
+++ b/KeywordSearch/nbproject/project.xml
@@ -115,6 +115,10 @@
                 <runtime-relative-path>ext/commons-lang-2.4.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/commons-lang-2.4.jar</binary-origin>
             </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/log4j-1.2.17.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/log4j-1.2.17.jar</binary-origin>
+            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/jcl-over-slf4j-1.6.4.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/jcl-over-slf4j-1.6.4.jar</binary-origin>
@@ -215,14 +219,14 @@
                 <runtime-relative-path>ext/boilerpipe-1.1.0.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/boilerpipe-1.1.0.jar</binary-origin>
             </class-path-extension>
-            <class-path-extension>
-                <runtime-relative-path>ext/jericho-html-3.2.jar</runtime-relative-path>
-                <binary-origin>release/modules/ext/jericho-html-3.2.jar</binary-origin>
-            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/org.osgi.compendium-4.0.0.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/org.osgi.compendium-4.0.0.jar</binary-origin>
             </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/slf4j-api-1.7.2.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/slf4j-api-1.7.2.jar</binary-origin>
+            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/jempbox-1.7.0.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/jempbox-1.7.0.jar</binary-origin>
@@ -279,14 +283,14 @@
                 <runtime-relative-path>ext/poi-scratchpad-3.8.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/poi-scratchpad-3.8.jar</binary-origin>
             </class-path-extension>
-            <class-path-extension>
-                <runtime-relative-path>ext/log4j-1.2.16.jar</runtime-relative-path>
-                <binary-origin>release/modules/ext/log4j-1.2.16.jar</binary-origin>
-            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/poi-ooxml-schemas-3.8.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/poi-ooxml-schemas-3.8.jar</binary-origin>
             </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/jericho-html-3.3.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/jericho-html-3.3.jar</binary-origin>
+            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/bcprov-jdk15-1.45.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/bcprov-jdk15-1.45.jar</binary-origin>
@@ -299,10 +303,6 @@
                 <runtime-relative-path>ext/apache-mime4j-dom-0.7.2.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/apache-mime4j-dom-0.7.2.jar</binary-origin>
             </class-path-extension>
-            <class-path-extension>
-                <runtime-relative-path>ext/slf4j-api-1.6.4.jar</runtime-relative-path>
-                <binary-origin>release/modules/ext/slf4j-api-1.6.4.jar</binary-origin>
-            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/geronimo-stax-api_1.0_spec-1.0.1.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/geronimo-stax-api_1.0_spec-1.0.1.jar</binary-origin>

From 40418148797bbad989442ee49567038a2e71b8e8 Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Fri, 28 Dec 2012 16:09:56 -0500
Subject: [PATCH 2/8] remove explicit optimize - not needed, and taking too
 much time to open case / core

---
 .../src/org/sleuthkit/autopsy/keywordsearch/Server.java    | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
index 42bcc7df63..6557910fa7 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
@@ -772,13 +772,6 @@ public class Server {
             this.name = name;
 
             this.solrCore = new HttpSolrServer(solrUrl + "/" + name);
-            try {
-                solrCore.optimize(true, true);
-            } catch (SolrServerException ex) {
-                logger.log(Level.SEVERE, "Error setting optimize on solr core", ex);
-            } catch (IOException ex) {
-                logger.log(Level.SEVERE, "Error setting optimize on solr core", ex);
-            }
 
             //TODO test these settings
             //solrCore.setSoTimeout(1000 * 60);  // socket read timeout, make large enough so can index larger files

From 55332e00769c2b3c467fb0b90cc9a9b794c2a0c1 Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Fri, 28 Dec 2012 18:16:39 -0500
Subject: [PATCH 3/8] AUT-732 	Text view exception fix Solr escaping html
 when generating highlight text, set html encoder to default

---
 KeywordSearch/release/solr/solr/conf/solrconfig.xml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/KeywordSearch/release/solr/solr/conf/solrconfig.xml b/KeywordSearch/release/solr/solr/conf/solrconfig.xml
index 3a743f3437..194c465f3b 100644
--- a/KeywordSearch/release/solr/solr/conf/solrconfig.xml
+++ b/KeywordSearch/release/solr/solr/conf/solrconfig.xml
@@ -903,14 +903,16 @@
 		
        <!-- Highlighting defaults -->
        <str name="hl">on</str>
-       <str name="hl.fl">content features title name</str>
+       <str name="hl.fl">text content file_name</str>
        <str name="hl.encoder">html</str>
+	   <str name="f.name.hl.fragsize">0</str>
+       <str name="f.name.hl.alternateField">file_name</str>
+	   
        <str name="hl.simple.pre">&lt;b&gt;</str>
        <str name="hl.simple.post">&lt;/b&gt;</str>
        <str name="f.title.hl.fragsize">0</str>
        <str name="f.title.hl.alternateField">title</str>
-       <str name="f.name.hl.fragsize">0</str>
-       <str name="f.name.hl.alternateField">name</str>
+       
        <str name="f.content.hl.snippets">3</str>
        <str name="f.content.hl.fragsize">200</str>
        <str name="f.content.hl.alternateField">content</str>
@@ -1546,10 +1548,12 @@
 
       <!-- Configure the standard encoder -->
       <encoder name="html" 
+			   default="true"
                class="solr.highlight.HtmlEncoder" />
 
       <!-- Configure the standard fragListBuilder -->
       <fragListBuilder name="simple" 
+					   default="true"
                        class="solr.highlight.SimpleFragListBuilder"/>
       
       <!-- Configure the single fragListBuilder -->

From c0f59153b8fe27da7310adf356611dfa830da46a Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Fri, 28 Dec 2012 18:17:32 -0500
Subject: [PATCH 4/8] enclose highlight text in html tags for correctness

---
 .../autopsy/keywordsearch/HighlightedMatchesSource.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
index 0443704f39..537eb996fe 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
@@ -357,7 +357,7 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
                 highlightedContent = insertAnchors(highlightedContent);
 
 
-                return "<pre>" + highlightedContent + "</pre>";
+                return "<html><pre>" + highlightedContent + "</pre></html>";
             }
         } catch (NoOpenCoreException ex) {
             logger.log(Level.WARNING, "Couldn't query markup for page: " + currentPage, ex);

From 241b7520b4514f1df578c6194a376fd0a7ed3ab9 Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Mon, 31 Dec 2012 10:57:41 -0500
Subject: [PATCH 5/8] optimize out profiled code - reduce num of allocations of
 result object

---
 .../autopsy/coreutils/StringExtract.java      | 40 +++++++++++++------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/StringExtract.java b/Core/src/org/sleuthkit/autopsy/coreutils/StringExtract.java
index bb8890d8a3..d6c43c24cc 100644
--- a/Core/src/org/sleuthkit/autopsy/coreutils/StringExtract.java
+++ b/Core/src/org/sleuthkit/autopsy/coreutils/StringExtract.java
@@ -52,6 +52,12 @@ public class StringExtract {
     private List<SCRIPT> enabledScripts;
     private boolean enableUTF8;
     private boolean enableUTF16;
+    
+    //stored and reused results
+    private final StringExtractResult resUTF16En1 = new StringExtractResult();
+    private final StringExtractResult resUTF16En2 = new StringExtractResult();
+    private final StringExtractResult resUTF8 = new StringExtractResult();
+    
     /**
      * supported scripts, can be overridden with enableScriptX methods
      */
@@ -202,22 +208,21 @@ public class StringExtract {
 
             //extract using all methods and see which one wins
             StringExtractResult resUTF16 = null;
+            boolean runUTF16 = false;
             if (enableUTF16 && curOffset % 2 == 0) {
-                StringExtractResult resUTF16En1 = extractUTF16(buff, len, curOffset, true);
-                StringExtractResult resUTF16En2 = extractUTF16(buff, len, curOffset, false);
+                runUTF16 = true;
+                extractUTF16(buff, len, curOffset, true, resUTF16En1);
+                extractUTF16(buff, len, curOffset, false, resUTF16En2);
                 resUTF16 = resUTF16En1.numChars > resUTF16En2.numChars ? resUTF16En1 : resUTF16En2;
             } 
             
-            //results.add(extractUTF8(buff, len, curOffset));
-            StringExtractResult resUTF8 = null;
-
             if (enableUTF8) {
-                resUTF8 = extractUTF8(buff, len, curOffset);
+                extractUTF8(buff, len, curOffset, resUTF8);
             }
 
             StringExtractResult resWin = null;
             if (enableUTF8 && enableUTF16) {
-                resWin = resUTF16 != null && resUTF16.numChars > resUTF8.numChars ? resUTF16 : resUTF8;
+                resWin = runUTF16 && resUTF16.numChars > resUTF8.numChars ? resUTF16 : resUTF8;
             } else if (enableUTF16){
                 resWin = resUTF16;
             }
@@ -250,7 +255,7 @@ public class StringExtract {
             }
         }
 
-        //build up the result
+        //build up the final result
         StringExtractResult res = new StringExtractResult();
         res.numBytes = processedBytes;
         res.numChars = curStringLen;
@@ -261,9 +266,9 @@ public class StringExtract {
         return res;
     }
 
-    private StringExtractResult extractUTF16(byte[] buff, int len, int offset, boolean endianSwap) {
-        StringExtractResult res = new StringExtractResult();
-
+    private StringExtractResult extractUTF16(byte[] buff, int len, int offset, boolean endianSwap, final StringExtractResult res) {
+        res.reset();
+        
         int curOffset = offset;
 
         final StringBuilder tempString = new StringBuilder();
@@ -352,8 +357,8 @@ public class StringExtract {
         return res;
     }
 
-    private StringExtractResult extractUTF8(byte[] buff, int len, int offset) {
-        StringExtractResult res = new StringExtractResult();
+    private StringExtractResult extractUTF8(byte[] buff, int len, int offset, final StringExtractResult res) {
+        res.reset();
 
         int curOffset = offset;
         int ch = 0; //character being extracted
@@ -608,6 +613,15 @@ public class StringExtract {
         int firstUnprocessedOff; ///< first byte past the last byte used in extraction, offset+numBytes for a single result, but we keep track of it for multiple extractions
         String textString; ///< the actual text string extracted, of numChars long
 
+        
+        void reset() {
+            offset = 0;
+            numBytes = 0;
+            numChars = 0;
+            firstUnprocessedOff = 0;
+            textString = null;
+        }
+        
         public int getFirstUnprocessedOff() {
             return firstUnprocessedOff;
         }

From 525a91d72c0799bf7a86c62aecfb8074db659f3e Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Mon, 31 Dec 2012 11:50:20 -0500
Subject: [PATCH 6/8] move module data cleanup to complete() and stop(). 
 Cleanup all data earlier.

---
 .../KeywordSearchIngestModule.java            | 120 ++++++++++--------
 1 file changed, 69 insertions(+), 51 deletions(-)

diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
index 366c5e08e4..3381ca7d5b 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
@@ -86,10 +86,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
             return time;
         }
     };
-    
     private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName());
     public static final String MODULE_NAME = "Keyword Search";
-    public static final String MODULE_DESCRIPTION = "Performs file indexing and periodic search using keywords and regular expressions in lists.";  
+    public static final String MODULE_DESCRIPTION = "Performs file indexing and periodic search using keywords and regular expressions in lists.";
     final public static String MODULE_VERSION = "1.0";
     private String args;
     private static KeywordSearchIngestModule instance = null;
@@ -116,10 +115,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
     private SleuthkitCase caseHandle = null;
     private static List<AbstractFileExtract> textExtractors;
     private static AbstractFileStringExtract stringExtractor;
-    
     private boolean initialized = false;
-    
-    
     private final GetIsFileKnownV getIsFileKnown = new GetIsFileKnownV();
     private KeywordSearchConfigurationPanel panel;
 
@@ -131,7 +127,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
 
     //private constructor to ensure singleton instance 
     private KeywordSearchIngestModule() {
-
     }
 
     /**
@@ -159,6 +154,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
 
         if (initialized == false) //error initializing indexing/Solr
         {
+            logger.log(Level.WARNING, "Skipping processing, module not initialized, file: " + abstractFile.getName());
             return ProcessResult.OK;
         }
 
@@ -170,12 +166,11 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
             indexer.indexFile(abstractFile, false);
             //notify depending module that keyword search (would) encountered error for this file
             return ProcessResult.ERROR;
-        }
-        else if (KeywordSearchSettings.getSkipKnown() && abstractFile.accept(getIsFileKnown) == true) {
+        } else if (KeywordSearchSettings.getSkipKnown() && abstractFile.accept(getIsFileKnown) == true) {
             //index meta-data only
             indexer.indexFile(abstractFile, false);
             return ProcessResult.OK;
-        } 
+        }
 
         if (processedFiles == false) {
             processedFiles = true;
@@ -186,13 +181,14 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
 
         //index the file and content (if the content is supported)
         indexer.indexFile(abstractFile, true);
-        
-        
+
+
         return ProcessResult.OK;
     }
-    
+
     /**
-     * Process content hierarchy and return true if content is a file and is set as known
+     * Process content hierarchy and return true if content is a file and is set
+     * as known
      */
     private class GetIsFileKnownV extends ContentVisitor.Default<Boolean> {
 
@@ -200,12 +196,11 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         protected Boolean defaultVisit(Content cntnt) {
             return false;
         }
-        
+
         @Override
         public Boolean visit(File file) {
             return file.getKnown() == FileKnown.KNOWN;
         }
-        
     }
 
     /**
@@ -246,7 +241,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         }
 
         //log number of files / chunks in index
-         //signal a potential change in number of indexed files
+        //signal a potential change in number of indexed files
         try {
             final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles();
             final int numIndexedChunks = KeywordSearch.getServer().queryNumIndexedChunks();
@@ -256,9 +251,10 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
             logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex);
         } catch (KeywordSearchModuleException se) {
             logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", se);
+        } finally {
+            cleanup();
         }
-        
-        
+
         //postSummary();
     }
 
@@ -288,6 +284,33 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         commit();
 
         //postSummary();
+
+        cleanup();
+    }
+
+    /**
+     * Common cleanup code when module stops or completes
+     */
+    private void cleanup() {
+        ingestStatus.clear();
+        currentResults.clear();
+        currentSearcher = null;
+        finalSearcher = null;
+
+        commitTimer.stop();
+        searchTimer.stop();
+        commitTimer = null;
+        searchTimer = null;
+
+        textExtractors.clear();
+        textExtractors = null;
+        stringExtractor = null;
+
+        keywords.clear();
+        keywordLists.clear();
+        keywordToList.clear();
+
+        initialized = false;
     }
 
     @Override
@@ -299,7 +322,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
     public String getDescription() {
         return MODULE_DESCRIPTION;
     }
-    
+
     @Override
     public String getVersion() {
         return MODULE_VERSION;
@@ -314,7 +337,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
     public void setArguments(String args) {
         this.args = args;
     }
-	
 
     /**
      * Initializes the module for new ingest run Sets up threads, timers,
@@ -326,20 +348,20 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         logger.log(Level.INFO, "init()");
         services = IngestServices.getDefault();
         initialized = false;
-        
+
         caseHandle = Case.getCurrentCase().getSleuthkitCase();
 
         ingester = Server.getIngester();
-        
+
         final Server server = KeywordSearch.getServer();
         try {
-            if (! server.isRunning()) {
+            if (!server.isRunning()) {
                 String msg = "Keyword search server was not properly initialized, cannot run keyword search ingest. ";
                 logger.log(Level.SEVERE, msg);
                 String details = msg + "Please try restarting the OS and the application";
                 services.postMessage(IngestMessage.createErrorMessage(++messageID, instance, msg, details));
                 return;
-                
+
             }
         } catch (KeywordSearchModuleException ex) {
             logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex);
@@ -352,8 +374,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         stringExtractor = new AbstractFileStringExtract();
         stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts());
         stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions());
-        
-        
+
+
         //log the scripts used for debugging
         final StringBuilder sbScripts = new StringBuilder();
         for (SCRIPT s : KeywordSearchSettings.getStringExtractScripts()) {
@@ -376,7 +398,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         initKeywords();
 
         if (keywords.isEmpty() || keywordLists.isEmpty()) {
-             services.postMessage(IngestMessage.createWarningMessage(++messageID, instance, "No keywords in keyword list.", "Only indexing will be done and and keyword search will be skipped (you can still add keyword lists using the Keyword Lists - Add to Ingest)."));
+            services.postMessage(IngestMessage.createWarningMessage(++messageID, instance, "No keywords in keyword list.", "Only indexing will be done and and keyword search will be skipped (you can still add keyword lists using the Keyword Lists - Add to Ingest)."));
         }
 
         processedFiles = false;
@@ -531,23 +553,24 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
     }
 
     /**
-     * Initialize the keyword search lists and associated keywords from the XML loader
-     * Use the lists to ingest that are set in the permanent XML configuration
+     * Initialize the keyword search lists and associated keywords from the XML
+     * loader Use the lists to ingest that are set in the permanent XML
+     * configuration
      */
     private void initKeywords() {
         addKeywordLists(null);
     }
-    
+
     /**
-     * If ingest is ongoing, this will add 
-     * additional keyword search lists to the ongoing ingest
-     * The lists to add may be temporary and not necessary set to be 
-     * added to ingest permanently in the XML configuration.
-     * The lists will be reset back to original (permanent configuration state)
-     * on the next ingest.
+     * If ingest is ongoing, this will add additional keyword search lists to
+     * the ongoing ingest The lists to add may be temporary and not necessary
+     * set to be added to ingest permanently in the XML configuration. The lists
+     * will be reset back to original (permanent configuration state) on the
+     * next ingest.
+     *
      * @param listsToAdd lists to add temporarily to the ongoing ingest
      */
-    void addKeywordLists(List<String>listsToAdd) {
+    void addKeywordLists(List<String> listsToAdd) {
         KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent();
 
         keywords.clear();
@@ -555,11 +578,11 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         keywordToList.clear();
 
         StringBuilder sb = new StringBuilder();
-        
+
         for (KeywordSearchList list : loader.getListsL()) {
             final String listName = list.getName();
             if (list.getUseForIngest() == true
-                    || (listsToAdd != null && listsToAdd.contains(listName)) ) {        
+                    || (listsToAdd != null && listsToAdd.contains(listName))) {
                 keywordLists.add(listName);
                 sb.append(listName).append(" ");
             }
@@ -571,9 +594,9 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
             }
 
         }
-        
+
         logger.log(Level.INFO, "Set new effective keyword lists: " + sb.toString());
-        
+
     }
 
     List<String> getKeywordLists() {
@@ -696,8 +719,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
             if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.VIRTUAL_DIR)) {
                 //skip indexing of virtual dirs (no content, no real name) - will index children files
                 return;
-            }
-            else if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
+            } else if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
                 fsContent = (FsContent) aFile;
             }
 
@@ -791,7 +813,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         private List<Keyword> keywords; //keywords to search
         private List<String> keywordLists; // lists currently being searched
         private Map<String, KeywordSearchList> keywordToList; //keyword to list name mapping
-    
         private ProgressHandle progress;
         private final Logger logger = Logger.getLogger(Searcher.class.getName());
         private boolean finalRun = false;
@@ -811,9 +832,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
         @Override
         protected Object doInBackground() throws Exception {
             if (finalRun) {
-               logger.log(Level.INFO, "Pending start of new (final) searcher");
-            }
-            else {
+                logger.log(Level.INFO, "Pending start of new (final) searcher");
+            } else {
                 logger.log(Level.INFO, "Pending start of new searcher");
             }
 
@@ -906,8 +926,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
                             if (this.isCancelled()) {
                                 logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: " + keywordQuery.getQuery());
                                 return null;
-                            }    
-                            
+                            }
+
                             List<ContentHit> contentHitsAll = newResults.get(hitTerm);
                             Map<AbstractFile, Integer> contentHitsFlattened = ContentHit.flattenResults(contentHitsAll);
                             for (final AbstractFile hitFile : contentHitsFlattened.keySet()) {
@@ -1079,8 +1099,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
                 //this is the final searcher
                 logger.log(Level.INFO, "The final searcher in this ingest done.");
                 finalSearcherDone = true;
-                //reset current resuls earlier to potentially garbage collect sooner
-                currentResults = new HashMap<Keyword, List<Long>>();
 
                 services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchIngestModule.instance, "Completed"));
             } else {

From c19993c48077e5d96e6c991aa9656b547ead1511 Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Wed, 2 Jan 2013 17:31:48 -0500
Subject: [PATCH 7/8] keyword search: - use FVH for snippets query for 3x
 snippet query performance - index 100k tokens per 1MB chunk instead of 10k,
 indexing all content - increase snippet length from 45 to 50 - add limit on
 regex query to be 20k results matching the regex - todo tune more - log how
 many fscontent hits on regex query - will use data to tune in future - minor
 code adjustments, debug queries if dev build, etc

---
 .../release/solr/solr/conf/schema.xml         | 10 +--
 .../HighlightedMatchesSource.java             | 66 ++++++++++++-------
 .../autopsy/keywordsearch/LuceneQuery.java    | 46 +++++++++----
 .../autopsy/keywordsearch/Server.java         |  8 +--
 .../keywordsearch/TermComponentQuery.java     | 16 ++++-
 5 files changed, 101 insertions(+), 45 deletions(-)

diff --git a/KeywordSearch/release/solr/solr/conf/schema.xml b/KeywordSearch/release/solr/solr/conf/schema.xml
index 333f0ed5f4..2059d0f1fb 100644
--- a/KeywordSearch/release/solr/solr/conf/schema.xml
+++ b/KeywordSearch/release/solr/solr/conf/schema.xml
@@ -209,7 +209,7 @@
     <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
       <analyzer>
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-        <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="10000"/>
+        <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="100000"/>
       </analyzer>
     </fieldType>
 	
@@ -222,7 +222,7 @@
       <analyzer type="index">
         <tokenizer class="solr.StandardTokenizerFactory"/>
         <!-- workaround to bug in Solr 4.0 to set LimitTokenCountFilterFactory maxTokenCount -->
-        <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="10000"/>
+        <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="100000"/>
         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
         <!-- in this example, we will only use synonyms at query time
         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
@@ -232,7 +232,7 @@
       <analyzer type="query">
         <tokenizer class="solr.StandardTokenizerFactory"/>
         <!-- workaround to bug in Solr 4.0 to set LimitTokenCountFilterFactory maxTokenCount -->
-        <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="10000"/>
+        <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="foo" foo="100000"/>
         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
         <filter class="solr.LowerCaseFilterFactory"/>
@@ -504,7 +504,7 @@
 
    <field name="id" type="string" indexed="true" stored="true" required="true" /> 
     <!-- The content field holds the text extracted by SolrCell -->
-   <field name="content" type="text_general" indexed="true" stored="true"/>
+   <field name="content" type="text_general" indexed="true" stored="true" termVectors="true" termPositions="true" termOffsets="true" />
    <!-- The strings field holds strings extracted from files that SolrCell doesn't support -->
    <field name="strings" type="text_general" indexed="true" stored="true"/>
    <field name="file_name" type="text_general" indexed="true" stored="true"/>
@@ -545,7 +545,7 @@
    
    <!-- field with white-space tokenized words for TermsComponent regex search (useful for fast search of IP addresses, URLs, certain phone numbers)
 		also be useful for Lucene based queries containing special characters-->
-   <field name="content_ws" type="text_ws" indexed="true" stored="false" multiValued="true"/> 
+   <field name="content_ws" type="text_ws" indexed="true" stored="false" /> 
 	
    <!-- Uncommenting the following will create a "timestamp" field using
         a default value of "NOW" to indicate when each document was indexed.
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
index 537eb996fe..6e45b3651d 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java
@@ -31,6 +31,7 @@ import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest.METHOD;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.sleuthkit.autopsy.coreutils.Version;
 import org.sleuthkit.autopsy.datamodel.HighlightLookup;
 import org.sleuthkit.autopsy.keywordsearch.KeywordQueryFilter.FilterType;
 import org.sleuthkit.datamodel.Content;
@@ -62,6 +63,7 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
     private Map<String, List<ContentHit>> hits = null; //original hits that may get passed in
     private String originalQuery = null; //or original query if hits are not available
     private boolean inited = false;
+    private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
 
     HighlightedMatchesSource(Content content, String keywordHitQuery, boolean isRegex) {
         this.content = content;
@@ -119,21 +121,17 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
         if (hasChunks) {
             //extract pages of interest, sorted
             final long contentId = content.getId();
-            
+
             if (hits == null) {
                 //special case, aka in case of dir tree, we don't know which chunks
                 //reperform search query for the content to get matching chunks info
                 KeywordSearchQuery chunksQuery = null;
-                
+
                 /**
-                Keyword keywordQuery = new Keyword(this.originalQuery, !isRegex);
-                if (this.isRegex)
-                {
-                    chunksQuery = new TermComponentQuery(keywordQuery);
-                } else {
-                    chunksQuery = new LuceneQuery(keywordQuery);
-                    chunksQuery.escape();
-                }
+                 * Keyword keywordQuery = new Keyword(this.originalQuery,
+                 * !isRegex); if (this.isRegex) { chunksQuery = new
+                 * TermComponentQuery(keywordQuery); } else { chunksQuery = new
+                 * LuceneQuery(keywordQuery); chunksQuery.escape(); }
                  */
                 String queryStr = KeywordSearchUtil.escapeLuceneQuery(this.keywordHitQuery);
                 if (isRegex) {
@@ -164,9 +162,11 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
             }
 
             //set page to first page having highlights
-            if (pagesSorted.isEmpty())
+            if (pagesSorted.isEmpty()) {
                 this.currentPage = 0;
-            else this.currentPage = pagesSorted.first();
+            } else {
+                this.currentPage = pagesSorted.first();
+            }
 
             for (Integer page : pagesSorted) {
                 hitsPages.put(page, 0); //unknown number of matches in the page
@@ -240,15 +240,17 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
 
     @Override
     public boolean hasNextItem() {
-        if (!this.pagesToHits.containsKey(currentPage))
+        if (!this.pagesToHits.containsKey(currentPage)) {
             return false;
+        }
         return this.pagesToHits.get(currentPage) < this.hitsPages.get(currentPage);
     }
 
     @Override
     public boolean hasPreviousItem() {
-        if (!this.pagesToHits.containsKey(currentPage))
+        if (!this.pagesToHits.containsKey(currentPage)) {
             return false;
+        }
         return this.pagesToHits.get(currentPage) > 1;
     }
 
@@ -274,8 +276,9 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
 
     @Override
     public int currentItem() {
-        if (!this.pagesToHits.containsKey(currentPage))
+        if (!this.pagesToHits.containsKey(currentPage)) {
             return 0;
+        }
         return pagesToHits.get(currentPage);
     }
 
@@ -307,6 +310,9 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
         }
 
         SolrQuery q = new SolrQuery();
+        q.setShowDebugInfo(DEBUG); //debug
+
+        String queryStr = null;
 
         if (isRegex) {
             StringBuilder sb = new StringBuilder();
@@ -318,13 +324,15 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
             if (group) {
                 sb.append("\"");
             }
-            q.setQuery(sb.toString());
+            queryStr = sb.toString();
         } else {
             //use default field, simplifies query
             //always force grouping/quotes
-            q.setQuery(KeywordSearchUtil.quoteQuery(highlightQuery));
+            queryStr = KeywordSearchUtil.quoteQuery(highlightQuery);
         }
 
+        q.setQuery(queryStr);
+
         final long contentId = content.getId();
 
         String contentIdStr = Long.toString(contentId);
@@ -332,13 +340,26 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
             contentIdStr += "_" + Integer.toString(this.currentPage);
         }
 
+
         final String filterQuery = Server.Schema.ID.toString() + ":" + contentIdStr;
         q.addFilterQuery(filterQuery);
         q.addHighlightField(highLightField); //for exact highlighting, try content_ws field (with stored="true" in Solr schema)
-        q.setHighlightSimplePre(HIGHLIGHT_PRE);
-        q.setHighlightSimplePost(HIGHLIGHT_POST);
-        q.setHighlightFragsize(0); // don't fragment the highlight
-        q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //analyze all content
+
+        //need to use original highlighter (as opposed to snippets), because FVH does not seem to support fragsize=0 to get entire content
+        //https://issues.apache.org/jira/browse/SOLR-1268?attachmentSortBy=dateTime
+
+        q.setHighlightSimplePre(HIGHLIGHT_PRE); //original highlighter only
+        q.setHighlightSimplePost(HIGHLIGHT_POST); //original highlighter only
+        q.setHighlightFragsize(0); // don't fragment the highlight, works with original highlighter only
+
+        //tune the highlighter
+        //q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one
+        //q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only
+        //q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only
+        //q.setParam("hl.fragListBuilder", "simple"); //makes sense for FastVectorHighlighter only
+
+        //docs says makes sense for the original Highlighter only, but not really
+        q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED);
 
         try {
             QueryResponse response = solrServer.query(q, METHOD.POST);
@@ -385,8 +406,9 @@ class HighlightedMatchesSource implements MarkupSource, HighlightLookup {
 
     @Override
     public int getNumberHits() {
-        if (!this.hitsPages.containsKey(this.currentPage))
+        if (!this.hitsPages.containsKey(this.currentPage)) {
             return 0;
+        }
         return this.hitsPages.get(this.currentPage);
     }
 
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
index ee7b73b3c6..e2b6ef91be 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
@@ -33,6 +33,7 @@ import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.coreutils.EscapeUtil;
+import org.sleuthkit.autopsy.coreutils.Version;
 import org.sleuthkit.datamodel.BlackboardArtifact;
 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
 import org.sleuthkit.datamodel.BlackboardAttribute;
@@ -50,11 +51,15 @@ public class LuceneQuery implements KeywordSearchQuery {
     private Keyword keywordQuery = null;
     private KeywordQueryFilter filter = null;
     private String field = null;
-    //use different highlight Solr fields for regex and literal search
+    private static final int MAX_RESULTS = 10000;
+    static final int SNIPPET_LENGTH = 50;
+    //can use different highlight schema fields for regex and literal search
     static final String HIGHLIGHT_FIELD_LITERAL = Server.Schema.CONTENT.toString();
     static final String HIGHLIGHT_FIELD_REGEX = Server.Schema.CONTENT.toString();
     //TODO use content_ws stored="true" in solr schema for perfect highlight hits
     //static final String HIGHLIGHT_FIELD_REGEX = Server.Schema.CONTENT_WS.toString()
+    
+    private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
 
     public LuceneQuery(Keyword keywordQuery) {
         this(keywordQuery.getQuery());
@@ -180,11 +185,11 @@ public class LuceneQuery implements KeywordSearchQuery {
         List<ContentHit> matches = new ArrayList<ContentHit>();
 
         boolean allMatchesFetched = false;
-        final int ROWS_PER_FETCH = 10000;
 
         final Server solrServer = KeywordSearch.getServer();
 
         SolrQuery q = new SolrQuery();
+        q.setShowDebugInfo(DEBUG); //debug
 
         //set query, force quotes/grouping around all literal queries
         final String groupedQuery = KeywordSearchUtil.quoteQuery(queryEscaped);
@@ -197,20 +202,20 @@ public class LuceneQuery implements KeywordSearchQuery {
         }
         
         q.setQuery(theQueryStr);
-        q.setRows(ROWS_PER_FETCH);
+        q.setRows(MAX_RESULTS);
         q.setFields(Server.Schema.ID.toString());
         if (filter != null) {
             q.addFilterQuery(filter.toString());
         }
 
-        for (int start = 0; !allMatchesFetched; start = start + ROWS_PER_FETCH) {
+        for (int start = 0; !allMatchesFetched; start = start + MAX_RESULTS) {
             q.setStart(start);
 
             try {
                 QueryResponse response = solrServer.query(q, METHOD.POST);
                 SolrDocumentList resultList = response.getResults();
                 long results = resultList.getNumFound();
-                allMatchesFetched = start + ROWS_PER_FETCH >= results;
+                allMatchesFetched = start + MAX_RESULTS >= results;
                 SleuthkitCase sc = null;
                 try {
                     sc = Case.getCurrentCase().getSleuthkitCase();
@@ -289,8 +294,6 @@ public class LuceneQuery implements KeywordSearchQuery {
      * @return 
      */
     public static String querySnippet(String query, long contentID, int chunkID, boolean isRegex, boolean group) throws NoOpenCoreException {
-        final int SNIPPET_LENGTH = 45;
-
         Server solrServer = KeywordSearch.getServer();
 
         String highlightField = null;
@@ -302,6 +305,8 @@ public class LuceneQuery implements KeywordSearchQuery {
 
         SolrQuery q = new SolrQuery();
 
+        String queryStr = null;
+        
         if (isRegex) {
             StringBuilder sb = new StringBuilder();
             sb.append(highlightField).append(":");
@@ -313,12 +318,14 @@ public class LuceneQuery implements KeywordSearchQuery {
                 sb.append("\"");
             }
 
-            q.setQuery(sb.toString());
+            queryStr = sb.toString();
         } else {
             //simplify query/escaping and use default field
             //always force grouping/quotes
-            q.setQuery(KeywordSearchUtil.quoteQuery(query));
+            queryStr = KeywordSearchUtil.quoteQuery(query);
         }
+        
+        q.setQuery(queryStr);
 
         String contentIDStr = null;
 
@@ -329,13 +336,28 @@ public class LuceneQuery implements KeywordSearchQuery {
         }
 
         String idQuery = Server.Schema.ID.toString() + ":" + contentIDStr;
+        q.setShowDebugInfo(DEBUG); //debug
         q.addFilterQuery(idQuery);
         q.addHighlightField(highlightField);
-        q.setHighlightSimplePre("&laquo;");
-        q.setHighlightSimplePost("&raquo;");
+        //q.setHighlightSimplePre("&laquo;"); //original highlighter only
+        //q.setHighlightSimplePost("&raquo;");  //original highlighter only
         q.setHighlightSnippets(1);
         q.setHighlightFragsize(SNIPPET_LENGTH);
-        q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //analyze all content SLOW! consider lowering
+        
+        
+        
+        //tune the highlighter
+        q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one
+        q.setParam("hl.tag.pre", "&laquo;"); //makes sense for FastVectorHighlighter only
+        q.setParam("hl.tag.post", "&laquo;"); //makes sense for FastVectorHighlighter only
+        q.setParam("hl.fragListBuilder", "simple"); //makes sense for FastVectorHighlighter only
+        
+         //Solr bug if fragCharSize is smaller than Query string, StringIndexOutOfBoundsException is thrown.
+        q.setParam("hl.fragCharSize", Integer.toString(queryStr.length())); //makes sense for FastVectorHighlighter only
+        
+        //docs says makes sense for the original Highlighter only, but not really
+        //analyze all content SLOW! consider lowering
+        q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); 
 
         try {
             QueryResponse response = solrServer.query(q);
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
index 6557910fa7..4aa82fcf29 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
@@ -118,7 +118,7 @@ public class Server {
             }
         },
     };
-    public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but option may not be supported.
+    public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
     //max content size we can send to Solr
     public static final long MAX_CONTENT_SIZE = 1L * 1024 * 1024 * 1024;
     private static final Logger logger = Logger.getLogger(Server.class.getName());
@@ -139,6 +139,7 @@ public class Server {
     static final int DEFAULT_SOLR_STOP_PORT = 34343;
     private int currentSolrServerPort = 0;
     private int currentSolrStopPort = 0;
+    private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
 
     public enum CORE_EVT_STATES {
 
@@ -257,7 +258,6 @@ public class Server {
         public void run() {
             InputStreamReader isr = new InputStreamReader(stream);
             BufferedReader br = new BufferedReader(isr);
-            final Version.Type builtType = Version.getBuildType();
             try {
                 OutputStreamWriter osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset());
                 BufferedWriter bw = new BufferedWriter(osw);
@@ -265,7 +265,7 @@ public class Server {
                 while (doRun && (line = br.readLine()) != null) {
                     bw.write(line);
                     bw.newLine();
-                    if (builtType == Version.Type.DEVELOPMENT) {
+                    if (DEBUG) {
                         //flush buffers if dev version for debugging
                         bw.flush();
                     }
@@ -292,7 +292,7 @@ public class Server {
                 String loggingPropertiesOpt = " -Djava.util.logging.config.file=";
                 String loggingPropertiesFilePath = instanceDir + File.separator + "conf" + File.separator;
 
-                if (Version.getBuildType().equals(Version.Type.DEVELOPMENT)) {
+                if (DEBUG) {
                     loggingPropertiesFilePath += "logging-development.properties";
                 } else {
                     loggingPropertiesFilePath += "logging-release.properties";
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
index 62a06faa83..89ed96d8f8 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
@@ -33,6 +33,7 @@ import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.response.TermsResponse;
 import org.apache.solr.client.solrj.response.TermsResponse.Term;
+import org.sleuthkit.autopsy.coreutils.Version;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.BlackboardArtifact;
 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
@@ -55,6 +56,9 @@ public class TermComponentQuery implements KeywordSearchQuery {
     private Keyword keywordQuery = null;
     private KeywordQueryFilter filter = null;
     private String field = null;
+    private static int MAX_TERMS_RESULTS = 20000;
+    
+    private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
 
     public TermComponentQuery(Keyword keywordQuery) {
         this.keywordQuery = keywordQuery;
@@ -217,9 +221,12 @@ public class TermComponentQuery implements KeywordSearchQuery {
         Map<String, List<ContentHit>> results = new HashMap<String, List<ContentHit>>();
 
         final SolrQuery q = createQuery();
+        q.setShowDebugInfo(DEBUG);
+        q.setTermsLimit(MAX_TERMS_RESULTS); 
         terms = executeQuery(q);
 
-
+        int resultSize = 0;
+        
         for (Term term : terms) {
             final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
 
@@ -232,7 +239,9 @@ public class TermComponentQuery implements KeywordSearchQuery {
                 Map<String, List<ContentHit>> subResults = filesQuery.performQuery();
                 Set<ContentHit> filesResults = new HashSet<ContentHit>();
                 for (String key : subResults.keySet()) {
-                    filesResults.addAll(subResults.get(key));
+                    List<ContentHit> keyRes = subResults.get(key);
+                    resultSize += keyRes.size();
+                    filesResults.addAll(keyRes);
                 }
                 results.put(term.getTerm(), new ArrayList<ContentHit>(filesResults));
             } catch (NoOpenCoreException e) {
@@ -243,6 +252,9 @@ public class TermComponentQuery implements KeywordSearchQuery {
             }
 
         }
+        
+        //TODO limit how many results we store, not to hit memory limits
+        logger.log(Level.INFO, "Regex # results: " + resultSize);
 
 
         return results;

From d10cf24a1a83766a164dc2bf9da26de986f0c683 Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Wed, 2 Jan 2013 17:36:33 -0500
Subject: [PATCH 8/8] update news

---
 NEWS.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/NEWS.txt b/NEWS.txt
index 5c35395456..558e576e8b 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -1,14 +1,17 @@
 ---------------- VERSION Current (dev) --------------
 New features:
-- Documented report module API
 
 
 Improvements:
+- Keyword search indexing and search speed improvements
 - Remake of reporting UI and functionality
 - Significant increase in reporting speed
+- Documented report module API
+- Upgrade to Solr4.0 / Tika 1.2
 
 
 Bugfixes:
+- Keyword search will index and search entire extracted content from files
 
 ---------------- VERSION 3.0.2 --------------