Fixed the iterator error introduced from the xlsx streamer approach.

Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java

@@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.tabulardatareader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -55,12 +54,14 @@ public class ExcelReader extends AbstractReader {
     private final String XLSXMimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
     private final String XLSMimeType = "application/vnd.ms-excel";
     private final String EMPTY_CELL_STRING = "";
+    private HashMap<String, Row> headerCache;
 
     public ExcelReader(AbstractFile file, String localDiskPath, String mimeType) 
             throws FileReaderInitException {
         super(file, localDiskPath);
         try {
             this.workbook = createWorkbook(localDiskPath, mimeType);
+            headerCache = new HashMap<>();
         } catch (IOException ex) {
             throw new FileReaderInitException(ex);
         }
@@ -92,13 +93,12 @@ public class ExcelReader extends AbstractReader {
                     throw new FileReaderInitException(e);
                 }
             case XLSXMimeType:
-                InputStream is = new FileInputStream(new File(localDiskPath));
                 //StreamingReader is part of the xlsx streamer dependency that creates
                 //a streaming version of XSSFWorkbook for reading (SXSSFWorkbook is only for writing
                 //large workbooks, not reading). This libary provides a workbook interface
                 //that is mostly identical to the poi workbook api, hence both the HSSFWorkbook
                 //and this can use the same functions below.
-                return StreamingReader.builder().rowCacheSize(500).bufferSize(4096).open(is);
+                return StreamingReader.builder().rowCacheSize(500).open(new File(localDiskPath));
             default:
                 throw new FileReaderInitException(String.format("Excel reader for mime "
                         + "type [%s] is not supported", mimeType));
@@ -126,7 +126,12 @@ public class ExcelReader extends AbstractReader {
      */
     @Override
     public List<Map<String, Object>> getRowsFromTable(String tableName) throws FileReaderException {
-        return getRowsFromTable(tableName, 0, getRowCountFromTable(tableName));
+        //Pad with + 1 because rows are zero index, thus a LastRowNum() (in getRowCountFromTable()) of 1
+        //indicates that there are records in 0 and 1 and so a total row count of 
+        //2. This also implies there is no way to determine if a workbook is empty,
+        //since a last row num of 0 doesnt differentiate between a record in 0 or 
+        //nothing in the workbook. Such a HSSF.
+        return getRowsFromTable(tableName, 0, getRowCountFromTable(tableName) + 1);
     }
     
     /**
@@ -145,27 +150,45 @@ public class ExcelReader extends AbstractReader {
     @Override
     public List<Map<String, Object>> getRowsFromTable(String tableName, 
             int offset, int numRowsToRead) throws FileReaderException {
+        //StreamingReader maintains the same pointer to a sheet rowIterator, so this
+        //call returns an iterator that could have already been iterated on instead
+        //of a fresh copy. We must cache the header value from the call to
+        //getTableSchemas as important information in the first row could have been
+        //missed.
         Iterator<Row> sheetIter = workbook.getSheet(tableName).rowIterator();
         List<Map<String, Object>> rowList = new ArrayList<>();
         
         int currRowCount = 0;
+        
+        //Read the header value as the header may be a row of data in the
+        //excel sheet
+        if(headerCache.containsKey(tableName)) {
+            Row header = headerCache.get(tableName);
+            if(currRowCount++ >= offset) {
+                rowList.add(getRowMap(tableName, header));
+            }
+        }
+        
         while(sheetIter.hasNext() && currRowCount < (offset + numRowsToRead)) {
             Row currRow = sheetIter.next();
-            if(currRowCount++ < offset) {
-                continue;
+            if(currRowCount++ >= offset) {
+                rowList.add(getRowMap(tableName, currRow));
             }
-            Map<String, Object> row = new HashMap<>();
-            for(Cell cell : currRow) {
-                String columnName = getColumnName(cell, tableName);
-                Object value = getCellValue(cell);
-                row.put(columnName, value);
-            }
-            rowList.add(row);
         }
         
         return rowList;
     }
     
+    private Map<String, Object> getRowMap(String tableName, Row row) {
+        Map<String, Object> rowMap = new HashMap<>();
+        for(Cell cell : row) {
+            String columnName = getColumnName(cell, tableName);
+            Object value = getCellValue(cell);
+            rowMap.put(columnName, value);
+        }
+        return rowMap;
+    }
+    
     /**
      * Returns the value of a given cell. The correct value function must be 
      * called on a cell depending on its type, hence the switch.
@@ -202,19 +225,14 @@ public class ExcelReader extends AbstractReader {
      * @return the name of the column the current cell lives in
      */
     private String getColumnName(Cell cell, String tableName) {
-        Iterator<Row> sheetIter = workbook.getSheet(tableName).rowIterator();
-        if(sheetIter.hasNext()) {
-            Row header = sheetIter.next();
+        if(headerCache.containsKey(tableName)) {
+            Row header = headerCache.get(tableName);
             Cell columnHeaderCell = header.getCell(cell.getRowIndex());
             if(columnHeaderCell == null) {
                 return EMPTY_CELL_STRING;
             }
             Object columnHeaderValue = getCellValue(columnHeaderCell);
-            if(columnHeaderValue instanceof String) {
-                return (String) columnHeaderValue;
-            } else {
-                return columnHeaderValue.toString();
-            }
+            return columnHeaderValue.toString();
         }
         //No header present
         return EMPTY_CELL_STRING;
@@ -222,6 +240,7 @@ public class ExcelReader extends AbstractReader {
 
     /**
      * Returns a map of sheet names to headers (header is in a comma-seperated string).
+     * Warning: Only call this ONCE per excel file.
      * 
      * @return A map of sheet names to header strings. 
      * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException 
@@ -234,6 +253,7 @@ public class ExcelReader extends AbstractReader {
             if(iterator.hasNext()) {
                 //Consume header
                 Row header = iterator.next();
+                headerCache.put(sheet.getSheetName(), header);
                 String headerStringFormat = StringUtils.join(header.cellIterator(), ", ");
                 tableSchemas.put(sheet.getSheetName(), headerStringFormat);
             }

Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java

@@ -82,7 +82,7 @@ public class SQLiteReader extends AbstractReader {
      * Searches for a meta file associated with the give SQLite database. If found,
      * copies the file to the local disk folder
      * 
-     * @param file file being processed
+     * @param sqliteFile file being processed
      * @param metaFileName name of meta file to look for
      * @throws NoCurrentCaseException Case has been closed.
      * @throws TskCoreException fileManager cannot find AbstractFile files.