diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 07c57b0fe6..0781a358dd 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -25,6 +25,7 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.Executors; @@ -74,6 +75,7 @@ import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.imagegallery.actions.UndoRedoManager; import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableFile; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableTagsManager; import org.sleuthkit.autopsy.imagegallery.datamodel.HashSetManager; @@ -200,11 +202,6 @@ public final class ImageGalleryController { Platform.runLater(() -> { stale.set(b); }); - try { - new PerCaseProperties(Case.getCurrentCaseThrows()).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.STALE, b.toString()); - } catch (NoCurrentCaseException ex) { - Logger.getLogger(ImageGalleryController.class.getName()).log(Level.WARNING, "Exception while getting open case."); //NON-NLS - } } public ReadOnlyBooleanProperty stale() { @@ -403,7 +400,10 @@ public final class ImageGalleryController { * */ public void rebuildDB() { - queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase)); + // queue a rebuild task for each stale data source + getStaleDataSourceIds().forEach((dataSourceObjId) -> { + queueDBTask(new CopyAnalyzedFiles(dataSourceObjId, instance, db, sleuthKitCase)); + }); } /** @@ -437,48 +437,57 @@ public final class ImageGalleryController { * @return true if datasources table is stale */ boolean isDataSourcesTableStale() { + return (getStaleDataSourceIds().isEmpty() == false); + } + + /** + * Returns a set of data source object ids that are stale. + * + * This includes any data sources already in the table, that are not in COMPLETE status, + * or any data sources that might have been added to the case, but are not in the datasources table. + * + * @return list of data source object ids that are stale. + */ + Set getStaleDataSourceIds() { + + Set staleDataSourceIds = new HashSet<>(); // no current case open to check if ((null == getDatabase()) || (null == getSleuthKitCase())) { - return false; + return staleDataSourceIds; } try { - Set knownDataSourceIds= getDatabase().getDataSourceIds(); + Map knownDataSourceIds= getDatabase().getDataSourceDbBuildStatus(); + List dataSources = getSleuthKitCase().getDataSources(); Set caseDataSourceIds = new HashSet<>(); dataSources.forEach((dataSource) -> { caseDataSourceIds.add(dataSource.getId()); }); - return !(knownDataSourceIds.containsAll(caseDataSourceIds) && caseDataSourceIds.containsAll(knownDataSourceIds)); + // collect all data sources already in the table, that are not yet COMPLETE + knownDataSourceIds.entrySet().stream().forEach((Map.Entry t) -> { + DrawableDbBuildStatusEnum status = t.getValue(); + if (DrawableDbBuildStatusEnum.COMPLETE != status) { + staleDataSourceIds.add(t.getKey()); + } + }); + + // collect any new data sources in the case. + caseDataSourceIds.forEach((Long id) -> { + if (!knownDataSourceIds.containsKey(id)) { + staleDataSourceIds.add(id); + } + }); + + return staleDataSourceIds; } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Image Gallery failed to check if datasources table is stale.", ex); - return false; - } - - } - - /** - * Update the datasources table in drawable DB. - * - */ - private void updateDataSourcesTable() { - // no current case open to update - if ((null == getDatabase()) || (null == getSleuthKitCase())) { - return; + return staleDataSourceIds; } - try { - List dataSources = getSleuthKitCase().getDataSources(); - dataSources.forEach((dataSource) -> { - getDatabase().insertDataSource(dataSource.getId()); - }); - } - catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Image Gallery failed to update data_sources table.", ex); - } } synchronized private void shutDownDBExecutor() { @@ -727,7 +736,10 @@ public final class ImageGalleryController { @NbBundle.Messages({"BulkTask.committingDb.status=committing image/video database", "BulkTask.stopCopy.status=Stopping copy to drawable db task.", "BulkTask.errPopulating.errMsg=There was an error populating Image Gallery database."}) - /* Base abstract class for various methods of copying data into the Image gallery DB */ + /** + * Base abstract class for various methods of copying image files data, + * for a given data source, into the Image gallery DB. + */ abstract static private class BulkTransferTask extends BackgroundTask { static private final String FILE_EXTENSION_CLAUSE = @@ -739,27 +751,35 @@ public final class ImageGalleryController { "(mime_type LIKE '" //NON-NLS + String.join("' OR mime_type LIKE '", FileTypeUtils.getAllSupportedMimeTypes()) //NON-NLS + "') "; - - static final String DRAWABLE_QUERY = - //grab files with supported extension - "(" + FILE_EXTENSION_CLAUSE - //grab files with supported mime-types - + " OR " + MIMETYPE_CLAUSE //NON-NLS - //grab files with image or video mime-types even if we don't officially support them - + " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )"; //NON-NLS - + + final String DRAWABLE_QUERY; + final String DATASOURCE_CLAUSE; + final ImageGalleryController controller; final DrawableDB taskDB; final SleuthkitCase tskCase; + final long dataSourceObjId; ProgressHandle progressHandle; - private boolean taskCompletionStatus; - BulkTransferTask(ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) { + BulkTransferTask(long dataSourceObjId, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) { this.controller = controller; this.taskDB = taskDB; this.tskCase = tskCase; + this.dataSourceObjId = dataSourceObjId; + + DATASOURCE_CLAUSE = " (data_source_obj_id = " + dataSourceObjId + ") "; + + DRAWABLE_QUERY = + DATASOURCE_CLAUSE + + " AND ( " + + //grab files with supported extension + FILE_EXTENSION_CLAUSE + //grab files with supported mime-types + + " OR " + MIMETYPE_CLAUSE //NON-NLS + //grab files with image or video mime-types even if we don't officially support them + + " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )"; //NON-NLS } /** @@ -768,8 +788,16 @@ public final class ImageGalleryController { */ abstract void cleanup(boolean success); - abstract List getFiles() throws TskCoreException; - + /** + * Gets a list of files to process. + * + * @return list of files to process + * @throws TskCoreException + */ + List getFiles() throws TskCoreException { + return tskCase.findAllFilesWhere(DRAWABLE_QUERY); + } + abstract void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDBTransaction) throws TskCoreException; @Override @@ -786,8 +814,9 @@ public final class ImageGalleryController { final List files = getFiles(); progressHandle.switchToDeterminate(files.size()); - updateProgress(0.0); + taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS); + updateProgress(0.0); taskCompletionStatus = true; int workDone = 0; @@ -839,6 +868,9 @@ public final class ImageGalleryController { return; } finally { progressHandle.finish(); + if (taskCompletionStatus) { + taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.COMPLETE); + } updateMessage(""); updateProgress(-1.0); } @@ -862,23 +894,20 @@ public final class ImageGalleryController { @NbBundle.Messages({"CopyAnalyzedFiles.committingDb.status=committing image/video database", "CopyAnalyzedFiles.stopCopy.status=Stopping copy to drawable db task.", "CopyAnalyzedFiles.errPopulating.errMsg=There was an error populating Image Gallery database."}) - static private class CopyAnalyzedFiles extends BulkTransferTask { + private class CopyAnalyzedFiles extends BulkTransferTask { - CopyAnalyzedFiles(ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) { - super(controller, taskDB, tskCase); + CopyAnalyzedFiles(long dataSourceObjId, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) { + super(dataSourceObjId, controller, taskDB, tskCase); } @Override protected void cleanup(boolean success) { - // processFile will set success to fail if files are missing MIME types - controller.setStale(!success); + // at the end of the task, set the stale status based on the + // cumulative status of all data sources + controller.setStale(isDataSourcesTableStale()); } - @Override - List getFiles() throws TskCoreException { - controller.updateDataSourcesTable(); - return tskCase.findAllFilesWhere(DRAWABLE_QUERY); - } + @Override void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDbTransaction) throws TskCoreException { @@ -928,16 +957,13 @@ public final class ImageGalleryController { static private class PrePopulateDataSourceFiles extends BulkTransferTask { private static final Logger LOGGER = Logger.getLogger(PrePopulateDataSourceFiles.class.getName()); - - private final Content dataSource; - + /** * * @param dataSourceId Data source object ID */ - PrePopulateDataSourceFiles(Content dataSource, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) { - super(controller, taskDB, tskCase); - this.dataSource = dataSource; + PrePopulateDataSourceFiles(long dataSourceObjId, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) { + super(dataSourceObjId, controller, taskDB, tskCase); } @Override @@ -949,13 +975,6 @@ public final class ImageGalleryController { taskDB.insertFile(DrawableFile.create(f, false, false), tr, caseDBTransaction); } - @Override - List getFiles() throws TskCoreException { - long datasourceID = dataSource.getDataSource().getId(); - taskDB.insertDataSource(datasourceID); - return tskCase.findAllFilesWhere("data_source_obj_id = " + datasourceID + " AND " + DRAWABLE_QUERY); - } - @Override @NbBundle.Messages({"PrePopulateDataSourceFiles.prepopulatingDb.status=prepopulating image/video database",}) ProgressHandle getInitialProgressHandle() { @@ -1020,9 +1039,7 @@ public final class ImageGalleryController { "Unable to determine if file is drawable and not known. Not making any changes to DB. See the logs for details."); } } - } else { - setStale(true); - } + } } break; } @@ -1058,13 +1075,8 @@ public final class ImageGalleryController { if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { Content newDataSource = (Content) evt.getNewValue(); if (isListeningEnabled()) { - queueDBTask(new PrePopulateDataSourceFiles(newDataSource, ImageGalleryController.this, getDatabase(), getSleuthKitCase())); - } else { - setStale(true); - } - } - else { - setStale(true); + queueDBTask(new PrePopulateDataSourceFiles(newDataSource.getId(), ImageGalleryController.this, getDatabase(), getSleuthKitCase())); + } } break; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 4ae1903de6..01b20eed77 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.imagegallery; import java.nio.file.Path; import java.nio.file.Paths; -import org.apache.commons.lang3.StringUtils; import static org.apache.commons.lang3.StringUtils.isNotBlank; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; @@ -85,10 +84,7 @@ public class ImageGalleryModule { */ public static boolean isDrawableDBStale(Case c) { if (c != null) { - String stale = new PerCaseProperties(c).getConfigSetting(ImageGalleryModule.MODULE_NAME, PerCaseProperties.STALE); - - return ( ImageGalleryController.getDefault().isDataSourcesTableStale() || - (StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : false) ); + return ImageGalleryController.getDefault().isDataSourcesTableStale(); } else { return false; } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/PerCaseProperties.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/PerCaseProperties.java index d87b13e880..56811352f0 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/PerCaseProperties.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/PerCaseProperties.java @@ -40,8 +40,6 @@ class PerCaseProperties { public static final String ENABLED = "enabled"; //NON-NLS - public static final String STALE = "stale"; //NON-NLS - private final Case theCase; PerCaseProperties(Case c) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 6221de8947..99b5e2bd63 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -102,7 +102,7 @@ public final class DrawableDB { private final PreparedStatement insertHashHitStmt; - private final PreparedStatement insertDataSourceStmt; + private final PreparedStatement updateDataSourceStmt; private final PreparedStatement updateFileStmt; private final PreparedStatement insertFileStmt; @@ -149,6 +149,15 @@ public final class DrawableDB { private final SleuthkitCase tskCase; private final ImageGalleryController controller; + /** + * Enum to track Image gallery db rebuild status for a data source + */ + public enum DrawableDbBuildStatusEnum { + UNKNOWN, /// no known status + IN_PROGRESS, /// drawable db rebuild has been started for the data source + COMPLETE; /// drawable db rebuild is complete for the data source + } + //////////////general database logic , mostly borrowed from sleuthkitcase /** * Lock to protect against concurrent write accesses to case database and to @@ -210,9 +219,9 @@ public final class DrawableDB { "INSERT OR IGNORE INTO drawable_files (obj_id , path, name, created_time, modified_time, make, model, analyzed) " //NON-NLS + "VALUES (?,?,?,?,?,?,?,?)"); //NON-NLS - insertDataSourceStmt = prepareStatement( - "INSERT OR IGNORE INTO datasources (ds_obj_id) " //NON-NLS - + "VALUES (?)"); //NON-NLS + updateDataSourceStmt = prepareStatement( + "INSERT OR REPLACE INTO datasources (ds_obj_id, drawable_db_build_status) " //NON-NLS + + " VALUES (?,?)"); //NON-NLS removeFileStmt = prepareStatement("DELETE FROM drawable_files WHERE obj_id = ?"); //NON-NLS @@ -374,7 +383,8 @@ public final class DrawableDB { try (Statement stmt = con.createStatement()) { String sql = "CREATE TABLE if not exists datasources " //NON-NLS + "( id INTEGER PRIMARY KEY, " //NON-NLS - + " ds_obj_id integer UNIQUE NOT NULL)"; //NON-NLS + + " ds_obj_id integer UNIQUE NOT NULL, " + + " drawable_db_build_status VARCHAR(128) )"; //NON-NLS stmt.execute(sql); } catch (SQLException ex) { LOGGER.log(Level.SEVERE, "problem creating datasources table", ex); //NON-NLS @@ -763,20 +773,20 @@ public final class DrawableDB { /** - * Gets all data source object ids from datasources table + * Gets all data source object ids from datasources table, and their DrawableDbBuildStatusEnum * - * @return list of known data source object ids + * @return map of known data source object ids, and their db status */ - public Set getDataSourceIds() throws TskCoreException { + public Map getDataSourceDbBuildStatus() throws TskCoreException { Statement statement = null; ResultSet rs = null; - Set ret = new HashSet<>(); + Map map = new HashMap<>(); dbReadLock(); try { statement = con.createStatement(); - rs = statement.executeQuery("SELECT ds_obj_id FROM datasources "); //NON-NLS + rs = statement.executeQuery("SELECT ds_obj_id, drawable_db_build_status FROM datasources "); //NON-NLS while (rs.next()) { - ret.add(rs.getLong(1)); + map.put(rs.getLong("ds_obj_id"), DrawableDbBuildStatusEnum.valueOf(rs.getString("drawable_db_build_status"))); } } catch (SQLException e) { throw new TskCoreException("SQLException while getting data source object ids", e); @@ -797,24 +807,25 @@ public final class DrawableDB { } dbReadUnlock(); } - return ret; + return map; } /** - * Insert given data source object id into datasources table + * Insert/update given data source object id and it's DB rebuild status in the datasources table. * - * If the object id exists in the table already, it does nothing. + * If the object id exists in the table already, it updates the status * * @param dsObjectId data source object id to insert */ - public void insertDataSource(long dsObjectId) { + public void insertOrUpdateDataSource(long dsObjectId, DrawableDbBuildStatusEnum status ) { dbWriteLock(); try { - // "INSERT OR IGNORE/ INTO datasources (ds_obj_id)" - insertDataSourceStmt.setLong(1,dsObjectId); + // "INSERT OR REPLACE INTO datasources (ds_obj_id, drawable_db_build_status) " //NON-NLS + updateDataSourceStmt.setLong(1,dsObjectId); + updateDataSourceStmt.setString(2, status.name()); - insertDataSourceStmt.executeUpdate(); + updateDataSourceStmt.executeUpdate(); } catch (SQLException | NullPointerException ex) { LOGGER.log(Level.SEVERE, "failed to insert/update datasources table", ex); //NON-NLS } finally {