Preliminary international string extract streaming, incorporate into Ingest (using default LATIN_2 script for now)

Minor cleanup, use Charset class, update comments.

CoreUtils/src/org/sleuthkit/autopsy/coreutils/StringExtract.java

@@ -32,13 +32,12 @@ import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.S
 
 /**
  * Language and encoding aware utility to extract strings from stream of bytes
- * Currently supports UTF-16 LE, UTF-16 BE and UTF8 Latin, Cyrillic, Chinese, Arabic
+ * Currently supports UTF-16 LE, UTF-16 BE and UTF8 Latin, Cyrillic, Chinese,
+ * Arabic
  *
- * TODO:
- * - add streaming interface 
+ * TODO: 
  * - process control characters 
- * - testing: check non-printable common chars sometimes extracted 
- * - check if need UTF8 to UTF16 conversion 
+ * - testing: check non-printable common chars sometimes extracted (font?)
  * - handle tie better (when number of chars in result is equal)
  */
 public class StringExtract {
@@ -58,11 +57,7 @@ public class StringExtract {
      */
     private static final List<SCRIPT> SUPPORTED_SCRIPTS =
             Arrays.asList(
-            SCRIPT.LATIN_2 
-            ,SCRIPT.ARABIC 
-            , SCRIPT.CYRILLIC 
-            , SCRIPT.HAN
-    );
+            SCRIPT.LATIN_2, SCRIPT.ARABIC, SCRIPT.CYRILLIC, SCRIPT.HAN);
 
     /**
      * Initializes the StringExtract utility Sets enabled scripts to all
@@ -119,7 +114,7 @@ public class StringExtract {
         return enabledScripts.contains(script);
 
     }
-    
+
     public static List<SCRIPT> getSupportedScripts() {
         return SUPPORTED_SCRIPTS;
     }
@@ -648,7 +643,7 @@ public class StringExtract {
         public static int getScriptValue(SCRIPT script) {
             return script.ordinal();
         }
-        
+
         public static SCRIPT scriptForString(String scriptStringVal) {
             SCRIPT script = SCRIPT.valueOf(scriptStringVal);
             return script;

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileChunk.java

@@ -19,6 +19,7 @@
 
 package org.sleuthkit.autopsy.keywordsearch;
 
+import java.nio.charset.Charset;
 import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
 
 /**
@@ -50,9 +51,9 @@ class AbstractFileChunk {
         return Server.getChunkIdString(this.parent.getSourceFile().getId(), this.chunkID);
     }
 
-    public boolean index(Ingester ingester, byte[] content, long contentSize, ByteContentStream.Encoding encoding) throws IngesterException {
+    public boolean index(Ingester ingester, byte[] content, long contentSize, Charset indexCharset) throws IngesterException {
         boolean success = true;
-        ByteContentStream bcs = new ByteContentStream(content, contentSize, parent.getSourceFile(), encoding);
+        ByteContentStream bcs = new ByteContentStream(content, contentSize, parent.getSourceFile(), indexCharset);
         try {
             ingester.ingest(this, bcs, content.length);
             //logger.log(Level.INFO, "Ingesting string chunk: " + this.getName() + ": " + chunkID);

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileExtract.java

@@ -19,6 +19,7 @@
 
 package org.sleuthkit.autopsy.keywordsearch;
 
+import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
 import org.sleuthkit.datamodel.AbstractFile;
 
 /**
@@ -47,6 +48,13 @@ interface AbstractFileExtract {
      */
     boolean index(AbstractFile sourceFile) throws Ingester.IngesterException;
     
+    /**
+     * Sets the script to use for the extraction
+     * @param extractScript script to use
+     * @return true if extractor supports script - specific extraction, false otherwise
+     */
+    boolean setScript(SCRIPT extractScript);
+    
     /**
      * Determines if the extractor works only for specified types
      * is supportedTypes() or whether is a generic content extractor (such as string extractor)

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java

@@ -24,19 +24,20 @@ import java.io.Reader;
 import java.nio.charset.Charset;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
 import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.ReadContentInputStream;
 
 /**
- * Extractor of text from HTML supported AbstractFile content.
- * Extracted text is divided into chunks and indexed with Solr.
- * If HTML extraction succeeds, chunks are indexed with Solr.
+ * Extractor of text from HTML supported AbstractFile content. Extracted text is
+ * divided into chunks and indexed with Solr. If HTML extraction succeeds,
+ * chunks are indexed with Solr.
  */
 public class AbstractFileHtmlExtract implements AbstractFileExtract {
+
     private static final Logger logger = Logger.getLogger(AbstractFileHtmlExtract.class.getName());
-    private static final ByteContentStream.Encoding ENCODING = ByteContentStream.Encoding.UTF8;
-    static final Charset charset = Charset.forName(ENCODING.toString());
+    static final Charset outCharset = Server.DEFAULT_INDEXED_TEXT_CHARSET;
     static final int MAX_EXTR_TEXT_CHARS = 512 * 1024;
     private static final int SINGLE_READ_CHARS = 1024;
     private static final int EXTRA_CHARS = 128; //for whitespace
@@ -46,15 +47,20 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
     private AbstractFile sourceFile;
     private int numChunks = 0;
     private static final String UTF16BOM = "\uFEFF";
-    private static final String [] SUPPORTED_EXTENSIONS = {
+    private static final String[] SUPPORTED_EXTENSIONS = {
         "htm", "html", "xhtml", "shtml", "xhtm", "shtm", "css", "js", "php", "jsp"
     };
-    
+
     AbstractFileHtmlExtract() {
         this.service = KeywordSearchIngestService.getDefault();
         ingester = Server.getIngester();
     }
 
+    @Override
+    public boolean setScript(SCRIPT extractScript) {
+        return false;
+    }
+
     @Override
     public int getNumChunks() {
         return numChunks;
@@ -69,24 +75,24 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
     public boolean index(AbstractFile sourceFile) throws IngesterException {
         this.sourceFile = sourceFile;
         this.numChunks = 0; //unknown until indexing is done
-        
+
         boolean success = false;
         Reader reader = null;
-        
+
         final InputStream stream = new ReadContentInputStream(sourceFile);
-        
+
         try {
             // Parse the stream with Jericho
             JerichoParserWrapper jpw = new JerichoParserWrapper(stream);
             jpw.parse();
             reader = jpw.getReader();
-            
+
             // In case there is an exception or parse() isn't called
             if (reader == null) {
                 logger.log(Level.WARNING, "No reader available from HTML parser");
                 return false;
             }
-            
+
             success = true;
             long readSize;
             long totalRead = 0;
@@ -135,10 +141,10 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
                 extracted = sb.toString();
 
                 //converts BOM automatically to charSet encoding
-                byte[] encodedBytes = extracted.getBytes(charset);
+                byte[] encodedBytes = extracted.getBytes(outCharset);
                 AbstractFileChunk chunk = new AbstractFileChunk(this, this.numChunks + 1);
                 try {
-                    chunk.index(ingester, encodedBytes, encodedBytes.length, ENCODING);
+                    chunk.index(ingester, encodedBytes, encodedBytes.length, outCharset);
                     ++this.numChunks;
                 } catch (Ingester.IngesterException ingEx) {
                     success = false;
@@ -171,13 +177,13 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
                 logger.log(Level.WARNING, "Unable to close content reader from " + sourceFile.getId(), ex);
             }
         }
-        
+
         //after all chunks, ingest the parent file without content itself, and store numChunks
         ingester.ingest(this);
 
         return success;
     }
-    
+
     @Override
     public boolean isContentTypeSpecific() {
         return true;
@@ -186,12 +192,11 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
     @Override
     public boolean isSupported(AbstractFile file) {
         String fileNameLower = file.getName().toLowerCase();
-        for (int i = 0; i< SUPPORTED_EXTENSIONS.length; ++i) {
+        for (int i = 0; i < SUPPORTED_EXTENSIONS.length; ++i) {
             if (fileNameLower.endsWith(SUPPORTED_EXTENSIONS[i])) {
                 return true;
             }
         }
         return false;
     }
-    
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringContentStream.java

@@ -22,30 +22,28 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.Charset;
 import java.util.logging.Logger;
 import org.apache.solr.common.util.ContentStream;
-import org.sleuthkit.autopsy.keywordsearch.ByteContentStream.Encoding;
 import org.sleuthkit.datamodel.AbstractContent;
 import org.sleuthkit.datamodel.AbstractFile;
 
 /**
- * Converter from AbstractContent into String with specific encoding
- * Then, an adapter back to Solr' ContentStream (which is a specific InputStream), 
- * using the same encoding
+ * Wrapper over InputStream that implements ContentStream to feed to Solr.
  */
 public class AbstractFileStringContentStream implements ContentStream {
     //input
 
     private AbstractFile content;
-    private Encoding encoding;
+    private Charset charset;
     //converted
-    private AbstractFileStringStream stream;
+    private InputStream stream;
     private static Logger logger = Logger.getLogger(AbstractFileStringContentStream.class.getName());
 
-    public AbstractFileStringContentStream(AbstractFile content, ByteContentStream.Encoding encoding) {
+    public AbstractFileStringContentStream(AbstractFile content, Charset charset, InputStream inputStream) {
         this.content = content;
-        this.encoding = encoding;
-        this.stream = new AbstractFileStringStream(content, encoding);
+        this.charset = charset;
+        this.stream = inputStream;
     }
 
     public AbstractContent getSourceContent() {
@@ -54,7 +52,7 @@ public class AbstractFileStringContentStream implements ContentStream {
 
     @Override
     public String getContentType() {
-        return "text/plain;charset=" + encoding.toString();
+        return "text/plain;charset=" + charset.name();
     }
 
     @Override
@@ -87,7 +85,7 @@ public class AbstractFileStringContentStream implements ContentStream {
     @Override
     protected void finalize() throws Throwable {
         super.finalize();
-
+        
         stream.close();
     }
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java

@@ -20,8 +20,10 @@ package org.sleuthkit.autopsy.keywordsearch;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
 import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
 import org.sleuthkit.datamodel.AbstractFile;
 
@@ -44,25 +46,31 @@ class AbstractFileStringExtract implements AbstractFileExtract {
     //single static buffer for all extractions.  Safe, indexing can only happen in one thread
     private static final byte[] STRING_CHUNK_BUF = new byte[(int) MAX_STRING_CHUNK_SIZE];
     private static final int BOM_LEN = 3;
+    private static final Charset INDEX_CHARSET = Server.DEFAULT_INDEXED_TEXT_CHARSET;
     
-    //private static final StringExtract se = new StringExtract();
+    private static final SCRIPT DEFAULT_SCRIPT = SCRIPT.LATIN_2;
+    private SCRIPT extractScript;
 
     static {
         //prepend UTF-8 BOM to start of the buffer
         STRING_CHUNK_BUF[0] = (byte) 0xEF;
         STRING_CHUNK_BUF[1] = (byte) 0xBB;
         STRING_CHUNK_BUF[2] = (byte) 0xBF;
-        
-        //se.init();
-        
-        
     }
 
     public AbstractFileStringExtract() {
         this.service = KeywordSearchIngestService.getDefault();
-        ingester = Server.getIngester();
+        this.ingester = Server.getIngester();
+        this.extractScript = DEFAULT_SCRIPT;
     }
 
+    @Override
+    public boolean setScript(SCRIPT extractScript) {
+        this.extractScript = extractScript;
+        return true;
+    }
+    
+
     @Override
     public int getNumChunks() {
         return this.numChunks;
@@ -80,7 +88,9 @@ class AbstractFileStringExtract implements AbstractFileExtract {
         boolean success = false;
         
         //construct stream that extracts text as we read it
-        final InputStream stringStream = new AbstractFileStringStream(sourceFile, ByteContentStream.Encoding.UTF8);
+        //final InputStream stringStream = new AbstractFileStringStream(sourceFile, INDEX_CHARSET);
+        final InputStream stringStream = new AbstractFileStringIntStream(
+                sourceFile, extractScript, INDEX_CHARSET);
 
         try {
             success = true;
@@ -94,7 +104,7 @@ class AbstractFileStringExtract implements AbstractFileExtract {
                 AbstractFileChunk chunk = new AbstractFileChunk(this, this.numChunks + 1);
 
                 try {
-                    chunk.index(ingester, STRING_CHUNK_BUF, readSize + BOM_LEN, ByteContentStream.Encoding.UTF8);
+                    chunk.index(ingester, STRING_CHUNK_BUF, readSize + BOM_LEN, INDEX_CHARSET);
                     ++this.numChunks;
                 } catch (IngesterException ingEx) {
                     success = false;

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringIntStream.java

@@ -0,0 +1,166 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2012 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.keywordsearch;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.sleuthkit.autopsy.coreutils.StringExtract;
+import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractResult;
+import org.sleuthkit.datamodel.AbstractFile;
+import org.sleuthkit.datamodel.TskCoreException;
+
+/**
+ * Wrapper over StringExtract to provide streaming API Given AbstractFile
+ * object, extract international strings from the file and read output as a
+ * stream of UTF-8 strings as encoded bytes.
+ */
+public class AbstractFileStringIntStream extends InputStream {
+
+    private AbstractFile content;
+    private final byte[] oneCharBuf = new byte[1];
+    private StringExtract stringExtractor;
+    private static final int FILE_BUF_SIZE = 1024 * 1024;
+    private static final byte[] fileReadBuff = new byte[FILE_BUF_SIZE]; //NOTE: need to run all stream extraction in same thread
+    private int fileReadOffset = 0;
+    private byte[] convertBuff; //stores extracted string encoded as bytes, before returned to user
+    private int convertBuffOffset = 0; //offset to start returning data to user on next read()
+    private int bytesInConvertBuff = 0; //amount of data currently in the buffer
+    private boolean fileEOF = false; //if file has more bytes to read
+    private Charset outCharset;
+    private static final Logger logger = Logger.getLogger(AbstractFileStringIntStream.class.getName());
+
+    /**
+     * Constructs new stream object that does convertion from file, to extracted
+     * strings, then to byte stream, for specified script auto-detected encoding
+     * (UTF8, UTF16LE, UTF16BE), and specified output byte stream encoding
+     *
+     * @param content
+     * @param script
+     * @param outCharset
+     */
+    public AbstractFileStringIntStream(AbstractFile content, StringExtract.StringExtractUnicodeTable.SCRIPT script, Charset outCharset) {
+        this.content = content;
+        this.stringExtractor = new StringExtract();
+        this.stringExtractor.setEnabledScript(script);
+        this.outCharset = outCharset;
+    }
+
+    @Override
+    public int read() throws IOException {
+        final int read = read(oneCharBuf, 0, 1);
+        if (read == 1) {
+            return oneCharBuf[0];
+        } else {
+            return -1;
+        }
+
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        if (b == null) {
+            throw new NullPointerException();
+        } else if (off < 0 || len < 0 || len > b.length - off) {
+            throw new IndexOutOfBoundsException();
+        } else if (len == 0) {
+            return 0;
+        }
+
+        long fileSize = content.getSize();
+        if (fileSize == 0) {
+            return -1;
+        }
+        
+
+        //read and convert until user buffer full
+        //we have data if file can be read or when byteBuff has converted strings to return
+        int bytesToUser = 0; //returned to user so far
+        while (bytesToUser < len) {
+            //check if we have enough converted strings         
+            int remain = bytesInConvertBuff - convertBuffOffset;
+
+            if ((convertBuff == null || remain == 0) && !fileEOF && fileReadOffset < fileSize) {
+                try {
+                    //convert more strings, store in buffer
+                    //TODO read repeatadly to ensure we have entire max FILE_BUF_SIZE
+                    final long toRead = Math.min(FILE_BUF_SIZE, fileSize - fileReadOffset);
+                    int read = content.read(fileReadBuff, fileReadOffset, toRead);
+                    if (read == -1 || read == 0) {
+                        fileEOF = true;
+                    } else {
+                        fileReadOffset += read;
+                        if (fileReadOffset >= fileSize) {
+                            fileEOF = true;
+                        }
+
+                        //put converted string in convertBuff
+                        convert(read);
+                        remain = bytesInConvertBuff - convertBuffOffset;
+                    }
+                } catch (TskCoreException ex) {
+                    //Exceptions.printStackTrace(ex);
+                    fileEOF = true;
+                }
+            }
+
+            //nothing more to read, and no more bytes in convertBuff
+            if (convertBuff == null || remain == 0) {
+                if (fileEOF) {
+                    return bytesToUser > 0 ? bytesToUser : -1;
+                } else {
+                    //no strings extracted, try another read
+                    continue;
+                }
+            }
+
+            //return part or all of convert buff to user
+            final int toCopy = Math.min(remain, len - off);
+            System.arraycopy(convertBuff, convertBuffOffset, b, off, toCopy);
+            convertBuffOffset += toCopy;
+
+            //TODO ensure that total bytesToUser < len, and save for next read()
+            bytesToUser += toCopy;
+
+        }
+
+        return bytesToUser;
+    }
+
+    /**
+     * convert bytes in file buffer to string, and encode string in
+     * convertBuffer
+     *
+     * @param numBytes num bytes in the fileReadBuff
+     */
+    private void convert(int numBytes) {
+        StringExtractResult ser = stringExtractor.extract(fileReadBuff, numBytes, 0);
+        convertBuff = ser.getText().getBytes(outCharset);
+
+        //reset tracking vars
+        if (ser.getNumBytes() == 0) {
+            bytesInConvertBuff = 0;
+        } else {
+            bytesInConvertBuff = convertBuff.length;
+        }
+        convertBuffOffset = 0;
+    }
+}

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringStream.java

@@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearch;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.sleuthkit.autopsy.datamodel.DataConversion;
@@ -28,17 +29,20 @@ import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.TskException;
 
 /**
- * FsContent input string stream reader/converter
- * TODO should be encoding specific and detect UTF8, UTF16LE, UTF16BE
- * then process remainder of the string using detected encoding  
+ * AbstractFile input string stream reader/converter - given AbstractFile, 
+ * extract strings from it and return encoded bytes via read()
+ * 
+ * Note: the utility supports extraction of only LATIN script and UTF8, UTF16LE, UTF16BE encodings
+ * and uses a brute force encoding detection - it's fast but could apply multiple encodings on the same string.
+ * 
+ * For other script/languages support and better encoding detection use AbstractFileStringIntStream streaming class,
+ * which wraps around StringExtract extractor.
  */
 public class AbstractFileStringStream extends InputStream {
 
-    
     //args
     private AbstractFile content;
-    private String encoding;
-    
+    private Charset outputCharset;
     //internal data
     private long contentOffset = 0; //offset in fscontent read into curReadBuf
     private static final int READ_BUF_SIZE = 256;
@@ -60,26 +64,30 @@ public class AbstractFileStringStream extends InputStream {
 
     /**
      * Construct new string stream from FsContent
+     *
      * @param content to extract strings from
-     * @param encoding target encoding, currently UTF-8
-     * @param preserveOnBuffBoundary whether to preserve or split string on a buffer boundary. If false, will pack into read buffer up to max. possible, potentially splitting a string. If false, the string will be preserved for next read.
+     * @param outputCharset target encoding to index as
+     * @param preserveOnBuffBoundary whether to preserve or split string on a
+     * buffer boundary. If false, will pack into read buffer up to max.
+     * possible, potentially splitting a string. If false, the string will be
+     * preserved for next read.
      */
-    public AbstractFileStringStream(AbstractFile content, ByteContentStream.Encoding encoding, boolean preserveOnBuffBoundary) {
+    public AbstractFileStringStream(AbstractFile content, Charset outputCharset, boolean preserveOnBuffBoundary) {
         this.content = content;
-        this.encoding = encoding.toString();
+        this.outputCharset = outputCharset;
         //this.preserveOnBuffBoundary = preserveOnBuffBoundary;
         //logger.log(Level.INFO, "FILE: " + content.getParentPath() + "/" + content.getName());
     }
-    
+
     /**
-     * Construct new string stream from FsContent
-     * Do not attempt to fill entire read buffer if that would break a string
-     * 
+     * Construct new string stream from FsContent Do not attempt to fill entire
+     * read buffer if that would break a string
+     *
      * @param content to extract strings from
-     * @param encoding target encoding, currently UTF-8
+     * @param outCharset target charset to encode into bytes and index as, e.g. UTF-8
      */
-    public AbstractFileStringStream(AbstractFile content, ByteContentStream.Encoding encoding) {
-        this(content, encoding, false);
+    public AbstractFileStringStream(AbstractFile content, Charset outCharset) {
+        this(content, outCharset, false);
     }
 
     @Override
@@ -100,7 +108,7 @@ public class AbstractFileStringStream extends InputStream {
         if (isEOF) {
             return -1;
         }
-        
+
 
         if (stringAtTempBoundary) {
             //append entire temp string residual from previous read()
@@ -113,8 +121,8 @@ public class AbstractFileStringStream extends InputStream {
 
         boolean singleConsecZero = false; //preserve the current sequence of chars if 1 consecutive zero char
         int newCurLen = curStringLen + tempStringLen;
-        
-        
+
+
         while (newCurLen < len) {
             //need to extract more strings
             if (readBufOffset > bytesInReadBuf - 1) {
@@ -248,23 +256,18 @@ public class AbstractFileStringStream extends InputStream {
     //copy currently extracted string to user buffer
     //and reset for next read() call
     private int copyToReturn(byte[] b, int off, long len) {
-        try {
-            final String curStringS = curString.toString();
-            //logger.log(Level.INFO, curStringS);
-            byte[] stringBytes = curStringS.getBytes(encoding);
-            System.arraycopy(stringBytes, 0, b, off, Math.min(curStringLen, (int) len));
-            //logger.log(Level.INFO, curStringS);
-            //copied all string, reset
-            curString = new StringBuilder();
-            int ret = curStringLen;
-            curStringLen = 0;
-            return ret;
-        } catch (UnsupportedEncodingException ex) {
-            //should not happen
-            logger.log(Level.SEVERE, "Bad encoding string: " + encoding, ex);
-        }
 
-        return 0;
+        final String curStringS = curString.toString();
+        //logger.log(Level.INFO, curStringS);
+        byte[] stringBytes = curStringS.getBytes(outputCharset);
+        System.arraycopy(stringBytes, 0, b, off, Math.min(curStringLen, (int) len));
+        //logger.log(Level.INFO, curStringS);
+        //copied all string, reset
+        curString = new StringBuilder();
+        int ret = curStringLen;
+        curStringLen = 0;
+        return ret;
+
     }
 
     @Override

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java

@@ -27,9 +27,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.sleuthkit.autopsy.ingest.IngestServiceAbstractFile;
@@ -37,8 +34,7 @@ import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.ReadContentInputStream;
 import org.apache.tika.Tika;
 import org.apache.tika.metadata.Metadata;
-import org.sleuthkit.autopsy.keywordsearch.ByteContentStream.Encoding;
-import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
+import org.sleuthkit.autopsy.coreutils.StringExtract;
 
 /**
  * Extractor of text from TIKA supported AbstractFile content. Extracted text is
@@ -53,8 +49,7 @@ import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
 public class AbstractFileTikaTextExtract implements AbstractFileExtract {
 
     private static final Logger logger = Logger.getLogger(IngestServiceAbstractFile.class.getName());
-    private static final Encoding ENCODING = Encoding.UTF8;
-    static final Charset charset = Charset.forName(ENCODING.toString());
+    private static final Charset OUTPUT_CHARSET = Server.DEFAULT_INDEXED_TEXT_CHARSET;
     static final int MAX_EXTR_TEXT_CHARS = 512 * 1024;
     private static final int SINGLE_READ_CHARS = 1024;
     private static final int EXTRA_CHARS = 128; //for whitespace
@@ -80,6 +75,11 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
         //tika.setMaxStringLength(MAX_EXTR_TEXT_CHARS); //for getting back string only
     }
 
+    @Override
+    public boolean setScript(StringExtract.StringExtractUnicodeTable.SCRIPT extractScript) {
+        return false;
+    }
+
     @Override
     public int getNumChunks() {
         return numChunks;
@@ -94,7 +94,7 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
     public boolean index(AbstractFile sourceFile) throws Ingester.IngesterException {
         this.sourceFile = sourceFile;
         this.numChunks = 0; //unknown until indexing is done
-        
+
         boolean success = false;
         Reader reader = null;
 
@@ -196,10 +196,10 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
                 extracted = sb.toString();
 
                 //converts BOM automatically to charSet encoding
-                byte[] encodedBytes = extracted.getBytes(charset);
+                byte[] encodedBytes = extracted.getBytes(OUTPUT_CHARSET);
                 AbstractFileChunk chunk = new AbstractFileChunk(this, this.numChunks + 1);
                 try {
-                    chunk.index(ingester, encodedBytes, encodedBytes.length, ENCODING);
+                    chunk.index(ingester, encodedBytes, encodedBytes.length, OUTPUT_CHARSET);
                     ++this.numChunks;
                 } catch (Ingester.IngesterException ingEx) {
                     success = false;

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ByteContentStream.java

@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.Charset;
 import java.util.logging.Logger;
 import org.apache.solr.common.util.ContentStream;
 import org.sleuthkit.datamodel.AbstractContent;
@@ -33,38 +34,21 @@ import org.sleuthkit.datamodel.AbstractContent;
  */
 public class ByteContentStream implements ContentStream {   
     
-    public static enum Encoding {
-
-        UTF8 {
-
-            @Override
-            public String toString() {
-                return "UTF-8";
-            }
-        },
-        UTF16 {
-
-            @Override
-            public String toString() {
-                return "UTF-16";
-            }
-        },
-    };
     
     //input
     private byte[] content; //extracted subcontent
     private long contentSize;
     private AbstractContent aContent; //origin
-    private Encoding encoding;
+    private Charset charset; //output byte stream charset of encoded strings
     
     private InputStream stream;
 
     private static Logger logger = Logger.getLogger(ByteContentStream.class.getName());
 
-    public ByteContentStream(byte [] content, long contentSize, AbstractContent aContent, Encoding encoding) {
+    public ByteContentStream(byte [] content, long contentSize, AbstractContent aContent, Charset charset) {
         this.content = content;
         this.aContent = aContent;
-        this.encoding = encoding;
+        this.charset = charset;
         stream = new ByteArrayInputStream(content, 0, (int)contentSize);
     }
 
@@ -79,7 +63,7 @@ public class ByteContentStream implements ContentStream {
 
     @Override
     public String getContentType() {
-        return "text/plain;charset=" + encoding.toString();
+        return "text/plain;charset=" + charset.name();
     }
 
     @Override

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java

@@ -46,14 +46,9 @@ import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.ContentVisitor;
 import org.sleuthkit.datamodel.Directory;
 import org.sleuthkit.datamodel.File;
-import org.sleuthkit.datamodel.FileSystem;
 import org.sleuthkit.datamodel.FsContent;
-import org.sleuthkit.datamodel.Image;
 import org.sleuthkit.datamodel.LayoutFile;
 import org.sleuthkit.datamodel.ReadContentInputStream;
-import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM;
-import org.sleuthkit.datamodel.Volume;
-import org.sleuthkit.datamodel.VolumeSystem;
 
 /**
  * Handles indexing files on a Solr core.

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java

@@ -32,6 +32,7 @@ import java.io.OutputStreamWriter;
 import java.net.ConnectException;
 import java.net.MalformedURLException;
 import java.net.SocketException;
+import java.nio.charset.Charset;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.swing.AbstractAction;
@@ -123,6 +124,9 @@ class Server {
     public static final String CORE_EVT = "CORE_EVT";
     public static final char ID_CHUNK_SEP = '_';
     private String javaPath = "java";
+    
+    public static final Charset DEFAULT_INDEXED_TEXT_CHARSET = Charset.forName("UTF-8"); ///< default Charset to index text as
+    
     private static final int MAX_SOLR_MEM_MB = 512; //TODO set dynamically based on avail. system resources
     private Process curSolrProcess = null;