From 6c6dc65ab04fc1fc98b0c7d319f01a1205e92247 Mon Sep 17 00:00:00 2001 From: Raman Date: Fri, 14 Dec 2018 11:04:13 -0500 Subject: [PATCH 1/6] 1130: determine when a path group is analyzed. --- .../imagegallery/ImageGalleryController.java | 4 +- .../imagegallery/ImageGalleryModule.java | 1 + .../imagegallery/datamodel/DrawableDB.java | 118 ++++++++++++------ .../datamodel/grouping/GroupManager.java | 52 +++++++- 4 files changed, 134 insertions(+), 41 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 64bd3f9e61..8862dd7097 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -653,7 +653,7 @@ public final class ImageGalleryController { public void run() { try { DrawableFile drawableFile = DrawableFile.create(getFile(), true, false); - getTaskDB().updateFile(drawableFile); + getTaskDB().updateFile(drawableFile, true); } catch (TskCoreException | SQLException ex) { Logger.getLogger(UpdateFileTask.class.getName()).log(Level.SEVERE, "Error in update file task", ex); //NON-NLS } @@ -873,7 +873,7 @@ public final class ImageGalleryController { // NOTE: Files are being processed because they have the right MIME type, // so we do not need to worry about this calculating them if (FileTypeUtils.hasDrawableMIMEType(f)) { - taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction); + taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction, false); } //unsupported mimtype => analyzed but shouldn't include else { taskDB.removeFile(f.getId(), tr); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 9db57c336c..b322d8e333 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -378,6 +378,7 @@ public class ImageGalleryModule { * database. */ if (controller.isListeningEnabled()) { + controller.getGroupManager().resetLastUpdatedPathGroup(); DrawableDB drawableDb = controller.getDatabase(); if (drawableDb.getDataSourceDbBuildStatus(dataSourceObjId) == DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index c06935345d..aaa18ec160 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -588,6 +588,7 @@ public final class DrawableDB { + " data_source_obj_id integer DEFAULT 0, " + " value VARCHAR(255) not null, " //NON-NLS + " attribute VARCHAR(255) not null, " //NON-NLS + + " isAnalyzed integer DEFAULT 0, " + " UNIQUE(data_source_obj_id, value, attribute) )"; //NON-NLS tskCase.getCaseDbAccessManager().createTable(GROUPS_TABLENAME, tableSchema); @@ -829,6 +830,26 @@ public final class DrawableDB { } + /** + * Sets the isAnalysed state in the groups table for the given group. + * + * @param groupKey group key. + * @param isAnalyzed + * + * @throws TskCoreException + */ + public void markGroupAnalyzed(GroupKey groupKey, boolean isAnalyzed) throws TskCoreException { + + String updateSQL = String.format(" SET isAnalyzed = %d " + + " WHERE attribute = \'%s\' AND value = \'%s\' and data_source_obj_id = %d ", + isAnalyzed ? 1 : 0, + SleuthkitCase.escapeSingleQuotes(groupKey.getAttribute().attrName.toString()), + SleuthkitCase.escapeSingleQuotes(groupKey.getValueDisplayName()), + groupKey.getAttribute() == DrawableAttribute.PATH ? groupKey.getDataSourceObjId() : 0); + + tskCase.getCaseDbAccessManager().update(GROUPS_TABLENAME, updateSQL); + } + /** * Removes a file from the drawables databse. * @@ -857,13 +878,22 @@ public final class DrawableDB { } } - public void updateFile(DrawableFile f) throws TskCoreException, SQLException { + /** + * Updates the image file. + * + * @param f file to update. + * @param addGroups indicates whether or not to add groups for the file. + * + * @throws TskCoreException + * @throws SQLException + */ + public void updateFile(DrawableFile f, boolean addGroups) throws TskCoreException, SQLException { DrawableTransaction trans = null; CaseDbTransaction caseDbTransaction = null; try { trans = beginTransaction(); caseDbTransaction = tskCase.beginTransaction(); - updateFile(f, trans, caseDbTransaction); + updateFile(f, trans, caseDbTransaction, addGroups); caseDbTransaction.commit(); commitTransaction(trans, true); } catch (TskCoreException | SQLException ex) { @@ -885,29 +915,19 @@ public final class DrawableDB { } } - /** - * Insert basic file data (no groups) into the DB during pre-population - * phase - * - * @param f - * @param tr - * @param caseDbTransaction - */ - public void insertBasicFileData(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { - insertOrUpdateFile(f, tr, caseDbTransaction, false); - } - + /** * Update an existing entry (or make a new one) into the DB that includes * group information. Called when a file has been analyzed or during a bulk * rebuild * - * @param f - * @param tr + * @param f file to update + * @param tr * @param caseDbTransaction + * @param addGroups specifies whether or not to add groups for the file */ - public void updateFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { - insertOrUpdateFile(f, tr, caseDbTransaction, true); + public void updateFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction, boolean addGroups) { + insertOrUpdateFile(f, tr, caseDbTransaction, addGroups); } /** @@ -1285,29 +1305,50 @@ public final class DrawableDB { } } - public Boolean isGroupAnalyzed(GroupKey gk) throws SQLException, TskCoreException { - dbWriteLock(); - try { - if (isClosed()) { - throw new SQLException("The drawables database is closed"); - } - try (Statement stmt = con.createStatement()) { - // In testing, this method appears to be a lot faster than doing one large select statement - Set fileIDsInGroup = getFileIDsInGroup(gk); - for (Long fileID : fileIDsInGroup) { - ResultSet analyzedQuery = stmt.executeQuery("SELECT analyzed FROM drawable_files WHERE obj_id = " + fileID); //NON-NLS - while (analyzedQuery.next()) { - if (analyzedQuery.getInt(ANALYZED) == 0) { - return false; + /** + * Returns whether or not the given group is analyzed and ready to be viewed. + * + * @param groupKey group key. + * @return true if the group is analyzed. + * @throws SQLException + * @throws TskCoreException + */ + public Boolean isGroupAnalyzed(GroupKey groupKey) throws SQLException, TskCoreException { + + // Callback to process result of isAnalyzed query + class IsGroupAnalyzedQueryResultProcessor extends CompletableFuture implements CaseDbAccessQueryCallback { + + @Override + public void process(ResultSet resultSet) { + try { + if (resultSet != null) { + while (resultSet.next()) { + complete(resultSet.getInt("isAnalyzed") == 1 ? true: false); //NON-NLS; + return; } } - return true; // THIS APPEARS TO BE A BUG (see JIRA-1130), THE FOR LOOP EXECUTES AT MOST ONCE + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to get group isAnalyzed", ex); //NON-NLS } } - return false; - } finally { - dbWriteUnlock(); } + + IsGroupAnalyzedQueryResultProcessor queryResultProcessor = new IsGroupAnalyzedQueryResultProcessor(); + try { + String groupAnalyzedQueryStmt = String.format("isAnalyzed FROM " + GROUPS_TABLENAME + + " WHERE attribute = \'%s\' AND value = \'%s\' and data_source_obj_id = %d ", + SleuthkitCase.escapeSingleQuotes(groupKey.getAttribute().attrName.toString()), + SleuthkitCase.escapeSingleQuotes(groupKey.getValueDisplayName()), + groupKey.getAttribute() == DrawableAttribute.PATH ? groupKey.getDataSourceObjId() : 0); + + tskCase.getCaseDbAccessManager().select(groupAnalyzedQueryStmt, queryResultProcessor); + return queryResultProcessor.get(); + } catch (ExecutionException | InterruptedException | TskCoreException ex) { + String msg = String.format("Failed to get group isAnalyzed for group key %s", groupKey.getValueDisplayName()); //NON-NLS + logger.log(Level.WARNING, msg, ex); + } + + return false; } /** @@ -1497,8 +1538,9 @@ public final class DrawableDB { return; } - String insertSQL = String.format(" (data_source_obj_id, value, attribute) VALUES (%d, \'%s\', \'%s\')", - ds_obj_id, SleuthkitCase.escapeSingleQuotes(value), SleuthkitCase.escapeSingleQuotes(groupBy.attrName.toString())); + int isAnalyzed = (groupBy == DrawableAttribute.PATH) ? 0 : 1; + String insertSQL = String.format(" (data_source_obj_id, value, attribute, isAnalyzed) VALUES (%d, \'%s\', \'%s\', %d)", + ds_obj_id, SleuthkitCase.escapeSingleQuotes(value), SleuthkitCase.escapeSingleQuotes(groupBy.attrName.toString()), isAnalyzed); if (DbType.POSTGRESQL == tskCase.getDatabaseType()) { insertSQL += " ON CONFLICT DO NOTHING"; } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 82fc6f7469..6f64e4659c 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -107,6 +107,11 @@ public class GroupManager { private final ImageGalleryController controller; + /** + * Keeps track of the last path group + * - a change in path indicates the last path group is analyzed + */ + private GroupKey lastUpdatedPathGroup = null; /** * list of all analyzed groups */ @@ -238,7 +243,8 @@ public class GroupManager { setGroupBy(DrawableAttribute.PATH); setSortOrder(SortOrder.ASCENDING); setDataSource(null); - + resetLastUpdatedPathGroup(); + unSeenGroups.forEach(controller.getCategoryManager()::unregisterListener); unSeenGroups.clear(); analyzedGroups.forEach(controller.getCategoryManager()::unregisterListener); @@ -618,6 +624,8 @@ public class GroupManager { for (GroupKey gk : groupsForFile) { // see if a group has been created yet for the key DrawableGroup g = getGroupForKey(gk); + + checkForPathGroupChange(gk); addFileToGroup(g, gk, fileId); } } @@ -625,7 +633,49 @@ public class GroupManager { //we fire this event for all files so that the category counts get updated during initial db population controller.getCategoryManager().fireChange(updatedFileIDs, null); } + + /** + * Checks if the given path is different from the last updated path group. + * If so, the last updated path group is marked as analyzed + * + * @param groupKey + */ + private void checkForPathGroupChange(GroupKey groupKey) { + try { + if (groupKey.getAttribute() == DrawableAttribute.PATH) { + + if (this.lastUpdatedPathGroup == null) { + lastUpdatedPathGroup = groupKey; + } + else if (groupKey.getValue().toString().equalsIgnoreCase(this.lastUpdatedPathGroup.getValue().toString()) == false) { + // mark the last path group as analyzed + getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup, true); + popuplateIfAnalyzed(lastUpdatedPathGroup, null); + + lastUpdatedPathGroup = groupKey; + } + } + } + catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Error setting isAnalyzed status for group: %s", groupKey.getValue().toString()), ex); //NON-NLS + } + } + /** + * Resets the last updated path group, after marking the last path group as analyzed. + */ + public void resetLastUpdatedPathGroup() { + try { + if (lastUpdatedPathGroup != null) { + getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup, true); + popuplateIfAnalyzed(lastUpdatedPathGroup, null); + lastUpdatedPathGroup = null; + } + } + catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Error resetting last path group: %s", lastUpdatedPathGroup.getValue().toString()), ex); //NON-NLS + } + } /** * If the group is analyzed (or other criteria based on grouping) and should * be shown to the user, then add it to the appropriate data structures so From f26c2201c6a29393e57728db9d06cee4730741a3 Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 26 Dec 2018 14:18:18 -0500 Subject: [PATCH 2/6] 1130: Mark group as analyzed - Fixed hang and address review comments in previous commit - Handle marking groups as analyzed during rebuild --- .../imagegallery/ImageGalleryController.java | 7 +-- .../imagegallery/datamodel/DrawableDB.java | 51 +++++++++++-------- .../datamodel/grouping/GroupManager.java | 14 +++-- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 8862dd7097..5ec1a83eec 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -653,7 +653,7 @@ public final class ImageGalleryController { public void run() { try { DrawableFile drawableFile = DrawableFile.create(getFile(), true, false); - getTaskDB().updateFile(drawableFile, true); + getTaskDB().updateFile(drawableFile); } catch (TskCoreException | SQLException ex) { Logger.getLogger(UpdateFileTask.class.getName()).log(Level.SEVERE, "Error in update file task", ex); //NON-NLS } @@ -700,7 +700,8 @@ public final class ImageGalleryController { //grab files with supported mime-types + MIMETYPE_CLAUSE //NON-NLS //grab files with image or video mime-types even if we don't officially support them - + " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )"; //NON-NLS + + " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )" //NON-NLS + + " ORDER BY parent_path "; } /** @@ -873,7 +874,7 @@ public final class ImageGalleryController { // NOTE: Files are being processed because they have the right MIME type, // so we do not need to worry about this calculating them if (FileTypeUtils.hasDrawableMIMEType(f)) { - taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction, false); + taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction); } //unsupported mimtype => analyzed but shouldn't include else { taskDB.removeFile(f.getId(), tr); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 40ae345a3a..f35451933b 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -36,6 +36,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import static java.util.Objects.isNull; @@ -831,18 +832,18 @@ public final class DrawableDB { } /** - * Sets the isAnalysed state in the groups table for the given group. + * Sets the isAnalysed flag in the groups table for the given group to true. * * @param groupKey group key. - * @param isAnalyzed * * @throws TskCoreException */ - public void markGroupAnalyzed(GroupKey groupKey, boolean isAnalyzed) throws TskCoreException { + public void markGroupAnalyzed(GroupKey groupKey) throws TskCoreException { + String updateSQL = String.format(" SET isAnalyzed = %d " + " WHERE attribute = \'%s\' AND value = \'%s\' and data_source_obj_id = %d ", - isAnalyzed ? 1 : 0, + 1, SleuthkitCase.escapeSingleQuotes(groupKey.getAttribute().attrName.toString()), SleuthkitCase.escapeSingleQuotes(groupKey.getValueDisplayName()), groupKey.getAttribute() == DrawableAttribute.PATH ? groupKey.getDataSourceObjId() : 0); @@ -882,18 +883,17 @@ public final class DrawableDB { * Updates the image file. * * @param f file to update. - * @param addGroups indicates whether or not to add groups for the file. * * @throws TskCoreException * @throws SQLException */ - public void updateFile(DrawableFile f, boolean addGroups) throws TskCoreException, SQLException { + public void updateFile(DrawableFile f) throws TskCoreException, SQLException { DrawableTransaction trans = null; CaseDbTransaction caseDbTransaction = null; try { trans = beginTransaction(); caseDbTransaction = tskCase.beginTransaction(); - updateFile(f, trans, caseDbTransaction, addGroups); + updateFile(f, trans, caseDbTransaction); caseDbTransaction.commit(); commitTransaction(trans, true); } catch (TskCoreException | SQLException ex) { @@ -924,10 +924,9 @@ public final class DrawableDB { * @param f file to update * @param tr * @param caseDbTransaction - * @param addGroups specifies whether or not to add groups for the file */ - public void updateFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction, boolean addGroups) { - insertOrUpdateFile(f, tr, caseDbTransaction, addGroups); + public void updateFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { + insertOrUpdateFile(f, tr, caseDbTransaction, true); } /** @@ -1316,16 +1315,19 @@ public final class DrawableDB { public Boolean isGroupAnalyzed(GroupKey groupKey) throws SQLException, TskCoreException { // Callback to process result of isAnalyzed query - class IsGroupAnalyzedQueryResultProcessor extends CompletableFuture implements CaseDbAccessQueryCallback { + class IsGroupAnalyzedQueryResultProcessor implements CaseDbAccessQueryCallback { + private boolean isAnalyzed = false; + + boolean getIsAnalyzed() { + return isAnalyzed; + } + @Override public void process(ResultSet resultSet) { - try { - if (resultSet != null) { - while (resultSet.next()) { - complete(resultSet.getInt("isAnalyzed") == 1 ? true: false); //NON-NLS; - return; - } + try { + if (resultSet.next()) { + isAnalyzed = resultSet.getInt("isAnalyzed") == 1 ? true: false; } } catch (SQLException ex) { logger.log(Level.SEVERE, "Failed to get group isAnalyzed", ex); //NON-NLS @@ -1342,10 +1344,10 @@ public final class DrawableDB { groupKey.getAttribute() == DrawableAttribute.PATH ? groupKey.getDataSourceObjId() : 0); tskCase.getCaseDbAccessManager().select(groupAnalyzedQueryStmt, queryResultProcessor); - return queryResultProcessor.get(); - } catch (ExecutionException | InterruptedException | TskCoreException ex) { + return queryResultProcessor.getIsAnalyzed(); + } catch ( TskCoreException ex) { String msg = String.format("Failed to get group isAnalyzed for group key %s", groupKey.getValueDisplayName()); //NON-NLS - logger.log(Level.WARNING, msg, ex); + logger.log(Level.SEVERE, msg, ex); } return false; @@ -1817,8 +1819,13 @@ public final class DrawableDB { */ public class DrawableTransaction { - private final Set updatedFiles = new HashSet<>(); - private final Set removedFiles = new HashSet<>(); + // The files are processed ORDERED BY parent path + // We want to preserve that order here, so that we can detect a + // change in path, and thus mark the path group as analyzed + // Hence we use a LinkedHashSet here. + private final Set updatedFiles = new LinkedHashSet<>(); + private final Set removedFiles = new LinkedHashSet<>(); + private boolean completed; private DrawableTransaction() throws TskCoreException, SQLException { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 6f64e4659c..66fd1fd524 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -243,7 +243,6 @@ public class GroupManager { setGroupBy(DrawableAttribute.PATH); setSortOrder(SortOrder.ASCENDING); setDataSource(null); - resetLastUpdatedPathGroup(); unSeenGroups.forEach(controller.getCategoryManager()::unregisterListener); unSeenGroups.clear(); @@ -636,7 +635,14 @@ public class GroupManager { /** * Checks if the given path is different from the last updated path group. - * If so, the last updated path group is marked as analyzed + * + * The idea is that when the path of the files being processed changes, + * we have moved from one folder to the next, and the group for the + * previous PATH can be considered as analyzed and can be displayed. + * + * NOTE: this a close approximation for when all files in a folder have been processed, + * but there's some room for error - files may go down the ingest pipleline + * out of order or the events may not always arrive in the same order * * @param groupKey */ @@ -649,7 +655,7 @@ public class GroupManager { } else if (groupKey.getValue().toString().equalsIgnoreCase(this.lastUpdatedPathGroup.getValue().toString()) == false) { // mark the last path group as analyzed - getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup, true); + getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup); popuplateIfAnalyzed(lastUpdatedPathGroup, null); lastUpdatedPathGroup = groupKey; @@ -667,7 +673,7 @@ public class GroupManager { public void resetLastUpdatedPathGroup() { try { if (lastUpdatedPathGroup != null) { - getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup, true); + getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup); popuplateIfAnalyzed(lastUpdatedPathGroup, null); lastUpdatedPathGroup = null; } From e9dd0c25e55df92bada31232d11f4e34ef9ae3b8 Mon Sep 17 00:00:00 2001 From: Raman Date: Fri, 4 Jan 2019 22:51:49 -0500 Subject: [PATCH 3/6] 1171: Create IG table schema versions --- .../imagegallery/datamodel/DrawableDB.java | 321 +++++++++++++++++- 1 file changed, 319 insertions(+), 2 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index f35451933b..f33ba298c7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2013-2018 Basis Technology Corp. + * Copyright 2013-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -78,6 +78,7 @@ import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData.DbType; import org.sleuthkit.datamodel.TskDataException; +import org.sleuthkit.datamodel.VersionNumber; import org.sqlite.SQLiteJDBCLoader; /** @@ -98,6 +99,16 @@ public final class DrawableDB { private static final String GROUPS_TABLENAME = "image_gallery_groups"; //NON-NLS private static final String GROUPS_SEEN_TABLENAME = "image_gallery_groups_seen"; //NON-NLS + private static final String IG_DB_INFO_TABLE = "image_gallery_db_info"; + + private static final String IG_SCHEMA_MAJOR_VERSION_KEY = "IG_SCHEMA_MAJOR_VERSION"; + private static final String IG_SCHEMA_MINOR_VERSION_KEY = "IG_SCHEMA_MINOR_VERSION"; + private static final String IG_CREATION_SCHEMA_MAJOR_VERSION_KEY = "IG_CREATION_SCHEMA_MAJOR_VERSION"; + private static final String IG_CREATION_SCHEMA_MINOR_VERSION_KEY = "IG_CREATION_SCHEMA_MINOR_VERSION"; + + private static final VersionNumber IG_STARTING_SCHEMA_VERSION = new VersionNumber(1, 0, 0); // IG Schema Starting version + private static final VersionNumber IG_SCHEMA_VERSION = new VersionNumber(1, 1, 0); // IG Schema Current version + private PreparedStatement insertHashSetStmt; private List preparedStatements = new ArrayList<>(); @@ -217,7 +228,7 @@ public final class DrawableDB { dbWriteLock(); try { con = DriverManager.getConnection("jdbc:sqlite:" + dbPath.toString()); //NON-NLS - if (!initializeDBSchema() || !prepareStatements() || !initializeStandardGroups() || !initializeImageList()) { + if (!initializeDBSchema() || !upgradeDBSchema() || !prepareStatements() || !initializeStandardGroups() || !initializeImageList()) { close(); throw new TskCoreException("Failed to initialize drawables database for Image Gallery use"); //NON-NLS } @@ -391,6 +402,34 @@ public final class DrawableDB { } } + /** + * Checks if the specified table exists in Drawable DB + * + * @param tableName table to check + * @return true if the table exists in the database + * + * @throws SQLException + */ + private boolean doesTableExist(String tableName) throws SQLException { + ResultSet tableQueryResults = null; + boolean tableExists = false; + try (Statement stmt = con.createStatement()) { + tableQueryResults = stmt.executeQuery("SELECT name FROM sqlite_master WHERE type='table'"); //NON-NLS + while (tableQueryResults.next()) { + if (tableQueryResults.getString("name").equalsIgnoreCase(tableName)) { + tableExists = true; + break; + } + } + } + finally { + if (tableQueryResults != null) { + tableQueryResults.close(); + } + } + return tableExists; + } + private static void deleteDatabaseIfOlderVersion(Path dbPath) throws SQLException, IOException { if (Files.exists(dbPath)) { boolean hasDrawableFilesTable = false; @@ -475,6 +514,8 @@ public final class DrawableDB { private boolean initializeDBSchema() { dbWriteLock(); try { + boolean existingDB = true; + if (isClosed()) { logger.log(Level.SEVERE, "The drawables database is closed"); //NON-NLS return false; @@ -491,6 +532,31 @@ public final class DrawableDB { * Create tables in the drawables database. */ try (Statement stmt = con.createStatement()) { + + // Check if the database is a new or existing database + existingDB = doesTableExist("datasources"); + if (false == doesTableExist(IG_DB_INFO_TABLE)) { + try { + VersionNumber ig_creation_schema_version = existingDB + ? IG_STARTING_SCHEMA_VERSION + : IG_SCHEMA_VERSION; + + stmt.execute("CREATE TABLE IF NOT EXISTS " + IG_DB_INFO_TABLE + " (name TEXT PRIMARY KEY, value TEXT NOT NULL)"); + + // backfill creation schema ver + stmt.execute(String.format("INSERT INTO %s (name, value) VALUES ('%s', '%s')", IG_DB_INFO_TABLE, IG_CREATION_SCHEMA_MAJOR_VERSION_KEY, ig_creation_schema_version.getMajor() )); + stmt.execute(String.format("INSERT INTO %s (name, value) VALUES ('%s', '%s')", IG_DB_INFO_TABLE, IG_CREATION_SCHEMA_MINOR_VERSION_KEY, ig_creation_schema_version.getMinor() )); + + // set current schema ver: at DB initialization - current version is same as starting version + stmt.execute(String.format("INSERT INTO %s (name, value) VALUES ('%s', '%s')", IG_DB_INFO_TABLE, IG_SCHEMA_MAJOR_VERSION_KEY, ig_creation_schema_version.getMajor() )); + stmt.execute(String.format("INSERT INTO %s (name, value) VALUES ('%s', '%s')", IG_DB_INFO_TABLE, IG_SCHEMA_MINOR_VERSION_KEY, ig_creation_schema_version.getMinor() )); + + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to create ig_db_info table", ex); //NON-NLS + return false; + } + } + try { String sql = "CREATE TABLE IF NOT EXISTS datasources " //NON-NLS + "( id INTEGER PRIMARY KEY, " //NON-NLS @@ -583,6 +649,44 @@ public final class DrawableDB { * Create tables in the case database. */ String autogenKeyType = (DbType.POSTGRESQL == tskCase.getDatabaseType()) ? "BIGSERIAL" : "INTEGER"; + + try { + VersionNumber ig_creation_schema_version = existingDB + ? IG_STARTING_SCHEMA_VERSION + : IG_SCHEMA_VERSION; + + String tableSchema = "( id " + autogenKeyType + " PRIMARY KEY, " + + " name TEXT UNIQUE NOT NULL," + + " value TEXT NOT NULL )"; + tskCase.getCaseDbAccessManager().createTable(IG_DB_INFO_TABLE, tableSchema); + + // backfill creation version + String creationMajorVerSQL = String.format(" (name, value) VALUES ('%s', '%s')", IG_CREATION_SCHEMA_MAJOR_VERSION_KEY, ig_creation_schema_version.getMajor()); + String creationMinorVerSQL = String.format(" (name, value) VALUES ('%s', '%s')", IG_CREATION_SCHEMA_MINOR_VERSION_KEY, ig_creation_schema_version.getMinor()); + + // set current version - at the onset, current version is same as creation version + String currentMajorVerSQL = String.format(" (name, value) VALUES ('%s', '%s')", IG_SCHEMA_MAJOR_VERSION_KEY, ig_creation_schema_version.getMajor()); + String currentMinorVerSQL = String.format(" (name, value) VALUES ('%s', '%s')", IG_SCHEMA_MINOR_VERSION_KEY, ig_creation_schema_version.getMinor()); + + if (DbType.POSTGRESQL == tskCase.getDatabaseType()) { + creationMajorVerSQL += " ON CONFLICT DO NOTHING "; + creationMinorVerSQL += " ON CONFLICT DO NOTHING "; + + currentMajorVerSQL += " ON CONFLICT DO NOTHING "; + currentMinorVerSQL += " ON CONFLICT DO NOTHING "; + } + + tskCase.getCaseDbAccessManager().insert(IG_DB_INFO_TABLE, creationMajorVerSQL); + tskCase.getCaseDbAccessManager().insert(IG_DB_INFO_TABLE, creationMinorVerSQL); + + tskCase.getCaseDbAccessManager().insert(IG_DB_INFO_TABLE, currentMajorVerSQL); + tskCase.getCaseDbAccessManager().insert(IG_DB_INFO_TABLE, currentMinorVerSQL); + + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Failed to create ig_db_info table in Case database", ex); //NON-NLS + return false; + } + try { String tableSchema = "( group_id " + autogenKeyType + " PRIMARY KEY, " //NON-NLS @@ -622,6 +726,219 @@ public final class DrawableDB { } } + /** + * Gets the Schema version from DrawableDB + * + * @return image gallery schema version in DrawableDB + * @throws SQLException + * @throws TskCoreException + */ + private VersionNumber getDrawableDbIgSchemaVersion() throws SQLException, TskCoreException { + + Statement statement = con.createStatement(); + ResultSet resultSet = null; + + try { + int majorVersion = -1; + String majorVersionStr = null; + resultSet = statement.executeQuery(String.format("SELECT value FROM %s WHERE name='%s'", IG_DB_INFO_TABLE, IG_SCHEMA_MAJOR_VERSION_KEY)); + if (resultSet.next()) { + majorVersionStr = resultSet.getString("value"); + try { + majorVersion = Integer.parseInt(majorVersionStr); + } catch (NumberFormatException ex) { + throw new TskCoreException("Bad value for schema major version = " + majorVersionStr, ex); + } + } else { + throw new TskCoreException("Failed to read schema major version from ig_db_info table"); + } + + int minorVersion = -1; + String minorVersionStr = null; + resultSet = statement.executeQuery(String.format("SELECT value FROM %s WHERE name='%s'", IG_DB_INFO_TABLE, IG_SCHEMA_MINOR_VERSION_KEY)); + if (resultSet.next()) { + minorVersionStr = resultSet.getString("value"); + try { + minorVersion = Integer.parseInt(minorVersionStr); + } catch (NumberFormatException ex) { + throw new TskCoreException("Bad value for schema minor version = " + minorVersionStr, ex); + } + } else { + throw new TskCoreException("Failed to read schema minor version from ig_db_info table"); + } + + return new VersionNumber(majorVersion, minorVersion, 0 ); + } + finally { + if (resultSet != null) { + resultSet.close(); + } + if (statement != null) { + statement.close(); + } + } + } + + /** + * Gets the ImageGallery schema version from CaseDB + * + * @return image gallery schema version in CaseDB + * @throws SQLException + * @throws TskCoreException + */ + private VersionNumber getCaseDbIgSchemaVersion() throws TskCoreException { + + // Callback to process result of get version query + class GetSchemaVersionQueryResultProcessor implements CaseDbAccessQueryCallback { + + private int version = -1; + + int getVersion() { + return version; + } + + @Override + public void process(ResultSet resultSet) { + try { + if (resultSet.next()) { + String versionStr = resultSet.getString("value"); + try { + version = Integer.parseInt(versionStr); + } catch (NumberFormatException ex) { + logger.log(Level.SEVERE, "Bad value for version = " + versionStr, ex); + } + } else { + logger.log(Level.SEVERE, "Failed to get version"); + } + } + catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to get version", ex); //NON-NLS + } + } + } + + GetSchemaVersionQueryResultProcessor majorVersionResultProcessor = new GetSchemaVersionQueryResultProcessor(); + GetSchemaVersionQueryResultProcessor minorVersionResultProcessor = new GetSchemaVersionQueryResultProcessor(); + + String versionQueryTemplate = "value FROM %s WHERE name = \'%s\' "; + tskCase.getCaseDbAccessManager().select(String.format(versionQueryTemplate, IG_DB_INFO_TABLE, IG_SCHEMA_MAJOR_VERSION_KEY), majorVersionResultProcessor); + tskCase.getCaseDbAccessManager().select(String.format(versionQueryTemplate, IG_DB_INFO_TABLE, IG_SCHEMA_MINOR_VERSION_KEY), minorVersionResultProcessor); + + return new VersionNumber(majorVersionResultProcessor.getVersion(), minorVersionResultProcessor.getVersion(), 0); + } + + /** + * Updates the IG schema version in the Drawable DB + * + * @param version + * + * @throws SQLException + */ + private void updateDrawableDbIgSchemaVersion(VersionNumber version) throws SQLException { + + dbWriteLock(); + try { + Statement statement = con.createStatement(); + + // update schema version + statement.execute(String.format("UPDATE %s SET value = '%s' WHERE name = '%s'", IG_DB_INFO_TABLE, version.getMajor(), IG_SCHEMA_MAJOR_VERSION_KEY )); + statement.execute(String.format("UPDATE %s SET value = '%s' WHERE name = '%s'", IG_DB_INFO_TABLE, version.getMinor(), IG_SCHEMA_MINOR_VERSION_KEY )); + + statement.close(); + return; + } + finally { + dbWriteUnlock(); + } + } + + /** + * Updates the IG schema version in CaseDB + * + * @param version + * + * @throws SQLException + */ + private void updateCaseDbIgSchemaVersion(VersionNumber version) throws TskCoreException { + + String updateSQLTemplate = " SET value = %s WHERE name = '%s' "; + tskCase.getCaseDbAccessManager().update(IG_DB_INFO_TABLE, String.format(updateSQLTemplate, version.getMajor(), IG_SCHEMA_MAJOR_VERSION_KEY)); + tskCase.getCaseDbAccessManager().update(IG_DB_INFO_TABLE, String.format(updateSQLTemplate, version.getMinor(), IG_SCHEMA_MINOR_VERSION_KEY)); + } + + + /** + * Upgrades the DB schema. + * + * @return true if the upgrade is successful + * + * @throws SQLException + * + */ + private boolean upgradeDBSchema() throws TskCoreException, SQLException { + + // Read current version from the DBs + VersionNumber drawableDbIgSchemaVersion = getDrawableDbIgSchemaVersion(); + VersionNumber caseDbIgSchemaVersion = getCaseDbIgSchemaVersion(); + + // Upgrade Schema in both DrawableDB and CaseDB + caseDbIgSchemaVersion = upgradeCaseDbIgSchema1dot0TO1dot1(caseDbIgSchemaVersion); + drawableDbIgSchemaVersion = upgradeDrawableDbIgSchema1dot0TO1dot1(caseDbIgSchemaVersion); + + // update the versions in the tables + updateDrawableDbIgSchemaVersion(drawableDbIgSchemaVersion); + updateCaseDbIgSchemaVersion(caseDbIgSchemaVersion); + + return true; + } + + /** + * Upgrades IG tables in CaseDB from 1.0 to 1.1 + * Does nothing if the incoming version is not 1.0 + * + * @param currVersion version to upgrade from + * + * @return new version number + * @throws TskCoreException + */ + private VersionNumber upgradeCaseDbIgSchema1dot0TO1dot1(VersionNumber currVersion ) throws TskCoreException { + + if (currVersion.getMajor() != 1 || + currVersion.getMinor() != 0) { + return currVersion; + } + + // 1.0 -> 1.1 upgrade + // Add a 'isAnalyzed' column to groups table in CaseDB + String alterSQL = " ADD COLUMN isAnalyzed integer DEFAULT 1 "; //NON-NLS + if (false == tskCase.getCaseDbAccessManager().doesColumnExist(GROUPS_TABLENAME, "isAnalyzed")) { + tskCase.getCaseDbAccessManager().alterTable(GROUPS_TABLENAME, alterSQL); + } + + return new VersionNumber(1,1,0); + } + + /** + * Upgrades IG tables in DrawableDB from 1.0 to 1.1 + * Does nothing if the incoming version is not 1.0 + * + * @param currVersion version to upgrade from + * + * @return new version number + * @throws TskCoreException + */ + private VersionNumber upgradeDrawableDbIgSchema1dot0TO1dot1(VersionNumber currVersion ) throws TskCoreException { + + if (currVersion.getMajor() != 1 || + currVersion.getMinor() != 0) { + return currVersion; + } + + // There are no changes in DrawableDB schema in 1.0 -> 1.1 + + return new VersionNumber(1,1,0); + } + @Override protected void finalize() throws Throwable { /* From face09c9d1fac5ede252122e64c06cc97f77b5ba Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 8 Jan 2019 07:44:00 -0500 Subject: [PATCH 4/6] Guard lastUpdatedPathGroup for concurrent access. --- .../autopsy/imagegallery/datamodel/grouping/GroupManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 66fd1fd524..12fde98654 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -111,6 +111,7 @@ public class GroupManager { * Keeps track of the last path group * - a change in path indicates the last path group is analyzed */ + @GuardedBy("this") //NOPMD private GroupKey lastUpdatedPathGroup = null; /** * list of all analyzed groups From 31a7e4cfeb441a0a40182b940f70f201e023a5a2 Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 9 Jan 2019 11:39:40 -0500 Subject: [PATCH 5/6] Address review comments: - Perform schema upgrade as a transaction --- .../imagegallery/datamodel/DrawableDB.java | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index f33ba298c7..1094910b97 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -830,11 +830,16 @@ public final class DrawableDB { /** * Updates the IG schema version in the Drawable DB * - * @param version + * @param version new version number + * @param transaction transaction under which the update happens * * @throws SQLException */ - private void updateDrawableDbIgSchemaVersion(VersionNumber version) throws SQLException { + private void updateDrawableDbIgSchemaVersion(VersionNumber version, DrawableTransaction transaction) throws SQLException, TskCoreException { + + if (transaction == null) { + throw new TskCoreException("Schema version update must be done in a transaction"); + } dbWriteLock(); try { @@ -845,7 +850,6 @@ public final class DrawableDB { statement.execute(String.format("UPDATE %s SET value = '%s' WHERE name = '%s'", IG_DB_INFO_TABLE, version.getMinor(), IG_SCHEMA_MINOR_VERSION_KEY )); statement.close(); - return; } finally { dbWriteUnlock(); @@ -855,15 +859,16 @@ public final class DrawableDB { /** * Updates the IG schema version in CaseDB * - * @param version + * @param version new version number + * @param caseDbTransaction transaction to use to update the CaseDB * * @throws SQLException */ - private void updateCaseDbIgSchemaVersion(VersionNumber version) throws TskCoreException { + private void updateCaseDbIgSchemaVersion(VersionNumber version, CaseDbTransaction caseDbTransaction) throws TskCoreException { String updateSQLTemplate = " SET value = %s WHERE name = '%s' "; - tskCase.getCaseDbAccessManager().update(IG_DB_INFO_TABLE, String.format(updateSQLTemplate, version.getMajor(), IG_SCHEMA_MAJOR_VERSION_KEY)); - tskCase.getCaseDbAccessManager().update(IG_DB_INFO_TABLE, String.format(updateSQLTemplate, version.getMinor(), IG_SCHEMA_MINOR_VERSION_KEY)); + tskCase.getCaseDbAccessManager().update(IG_DB_INFO_TABLE, String.format(updateSQLTemplate, version.getMajor(), IG_SCHEMA_MAJOR_VERSION_KEY), caseDbTransaction); + tskCase.getCaseDbAccessManager().update(IG_DB_INFO_TABLE, String.format(updateSQLTemplate, version.getMinor(), IG_SCHEMA_MINOR_VERSION_KEY), caseDbTransaction); } @@ -882,13 +887,39 @@ public final class DrawableDB { VersionNumber caseDbIgSchemaVersion = getCaseDbIgSchemaVersion(); // Upgrade Schema in both DrawableDB and CaseDB - caseDbIgSchemaVersion = upgradeCaseDbIgSchema1dot0TO1dot1(caseDbIgSchemaVersion); - drawableDbIgSchemaVersion = upgradeDrawableDbIgSchema1dot0TO1dot1(caseDbIgSchemaVersion); - - // update the versions in the tables - updateDrawableDbIgSchemaVersion(drawableDbIgSchemaVersion); - updateCaseDbIgSchemaVersion(caseDbIgSchemaVersion); + CaseDbTransaction caseDbTransaction = tskCase.beginTransaction(); + DrawableTransaction transaction = beginTransaction(); + + try { + caseDbIgSchemaVersion = upgradeCaseDbIgSchema1dot0TO1dot1(caseDbIgSchemaVersion, caseDbTransaction); + drawableDbIgSchemaVersion = upgradeDrawableDbIgSchema1dot0TO1dot1(drawableDbIgSchemaVersion, transaction); + // update the versions in the tables + updateCaseDbIgSchemaVersion(caseDbIgSchemaVersion, caseDbTransaction ); + updateDrawableDbIgSchemaVersion(drawableDbIgSchemaVersion, transaction); + + caseDbTransaction.commit(); + caseDbTransaction = null; + commitTransaction(transaction, false); + transaction = null; + } + catch (TskCoreException | SQLException ex) { + if (null != caseDbTransaction) { + try { + caseDbTransaction.rollback(); + } catch (TskCoreException ex2) { + logger.log(Level.SEVERE, String.format("Failed to roll back case db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS + } + } + if (null != transaction) { + try { + rollbackTransaction(transaction); + } catch (SQLException ex2) { + logger.log(Level.SEVERE, String.format("Failed to roll back drawables db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS + } + } + throw ex; + } return true; } @@ -897,11 +928,12 @@ public final class DrawableDB { * Does nothing if the incoming version is not 1.0 * * @param currVersion version to upgrade from + * @param caseDbTransaction transaction to use for all updates * * @return new version number * @throws TskCoreException */ - private VersionNumber upgradeCaseDbIgSchema1dot0TO1dot1(VersionNumber currVersion ) throws TskCoreException { + private VersionNumber upgradeCaseDbIgSchema1dot0TO1dot1(VersionNumber currVersion, CaseDbTransaction caseDbTransaction ) throws TskCoreException { if (currVersion.getMajor() != 1 || currVersion.getMinor() != 0) { @@ -911,11 +943,10 @@ public final class DrawableDB { // 1.0 -> 1.1 upgrade // Add a 'isAnalyzed' column to groups table in CaseDB String alterSQL = " ADD COLUMN isAnalyzed integer DEFAULT 1 "; //NON-NLS - if (false == tskCase.getCaseDbAccessManager().doesColumnExist(GROUPS_TABLENAME, "isAnalyzed")) { - tskCase.getCaseDbAccessManager().alterTable(GROUPS_TABLENAME, alterSQL); + if (false == tskCase.getCaseDbAccessManager().columnExists(GROUPS_TABLENAME, "isAnalyzed", caseDbTransaction )) { + tskCase.getCaseDbAccessManager().alterTable(GROUPS_TABLENAME, alterSQL, caseDbTransaction); } - - return new VersionNumber(1,1,0); + return new VersionNumber(1,1,0); } /** @@ -923,11 +954,12 @@ public final class DrawableDB { * Does nothing if the incoming version is not 1.0 * * @param currVersion version to upgrade from + * @param transaction transaction to use for all updates * * @return new version number * @throws TskCoreException */ - private VersionNumber upgradeDrawableDbIgSchema1dot0TO1dot1(VersionNumber currVersion ) throws TskCoreException { + private VersionNumber upgradeDrawableDbIgSchema1dot0TO1dot1(VersionNumber currVersion, DrawableTransaction transaction ) throws TskCoreException { if (currVersion.getMajor() != 1 || currVersion.getMinor() != 0) { @@ -935,7 +967,6 @@ public final class DrawableDB { } // There are no changes in DrawableDB schema in 1.0 -> 1.1 - return new VersionNumber(1,1,0); } From 2a80c5733bb8ccb620f701a729b4626d9e513ca2 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 10 Jan 2019 11:00:12 -0500 Subject: [PATCH 6/6] Pulled in changes for 1130 (Is Group Analyzed) to keep all the related changes together on a single branch. Addressed review comments. --- .../imagegallery/ImageGalleryModule.java | 2 +- .../imagegallery/datamodel/DrawableDB.java | 18 ++++---- .../datamodel/grouping/GroupManager.java | 42 ++++++++++--------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 0e4e1b84e6..2038815236 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -383,7 +383,7 @@ public class ImageGalleryModule { * database. */ if (controller.isListeningEnabled()) { - controller.getGroupManager().resetLastUpdatedPathGroup(); + controller.getGroupManager().resetCurrentPathGroup(); DrawableDB drawableDb = controller.getDatabase(); if (drawableDb.getDataSourceDbBuildStatus(dataSourceObjId) == DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 1094910b97..6b508c2a4c 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -693,7 +693,7 @@ public final class DrawableDB { + " data_source_obj_id integer DEFAULT 0, " + " value VARCHAR(255) not null, " //NON-NLS + " attribute VARCHAR(255) not null, " //NON-NLS - + " isAnalyzed integer DEFAULT 0, " + + " is_analyzed integer DEFAULT 0, " + " UNIQUE(data_source_obj_id, value, attribute) )"; //NON-NLS tskCase.getCaseDbAccessManager().createTable(GROUPS_TABLENAME, tableSchema); @@ -942,8 +942,8 @@ public final class DrawableDB { // 1.0 -> 1.1 upgrade // Add a 'isAnalyzed' column to groups table in CaseDB - String alterSQL = " ADD COLUMN isAnalyzed integer DEFAULT 1 "; //NON-NLS - if (false == tskCase.getCaseDbAccessManager().columnExists(GROUPS_TABLENAME, "isAnalyzed", caseDbTransaction )) { + String alterSQL = " ADD COLUMN is_analyzed integer DEFAULT 1 "; //NON-NLS + if (false == tskCase.getCaseDbAccessManager().columnExists(GROUPS_TABLENAME, "is_analyzed", caseDbTransaction )) { tskCase.getCaseDbAccessManager().alterTable(GROUPS_TABLENAME, alterSQL, caseDbTransaction); } return new VersionNumber(1,1,0); @@ -1189,7 +1189,7 @@ public final class DrawableDB { public void markGroupAnalyzed(GroupKey groupKey) throws TskCoreException { - String updateSQL = String.format(" SET isAnalyzed = %d " + String updateSQL = String.format(" SET is_analyzed = %d " + " WHERE attribute = \'%s\' AND value = \'%s\' and data_source_obj_id = %d ", 1, SleuthkitCase.escapeSingleQuotes(groupKey.getAttribute().attrName.toString()), @@ -1675,17 +1675,17 @@ public final class DrawableDB { public void process(ResultSet resultSet) { try { if (resultSet.next()) { - isAnalyzed = resultSet.getInt("isAnalyzed") == 1 ? true: false; + isAnalyzed = resultSet.getInt("is_analyzed") == 1 ? true: false; } } catch (SQLException ex) { - logger.log(Level.SEVERE, "Failed to get group isAnalyzed", ex); //NON-NLS + logger.log(Level.SEVERE, "Failed to get group is_analyzed", ex); //NON-NLS } } } IsGroupAnalyzedQueryResultProcessor queryResultProcessor = new IsGroupAnalyzedQueryResultProcessor(); try { - String groupAnalyzedQueryStmt = String.format("isAnalyzed FROM " + GROUPS_TABLENAME + String groupAnalyzedQueryStmt = String.format("is_analyzed FROM " + GROUPS_TABLENAME + " WHERE attribute = \'%s\' AND value = \'%s\' and data_source_obj_id = %d ", SleuthkitCase.escapeSingleQuotes(groupKey.getAttribute().attrName.toString()), SleuthkitCase.escapeSingleQuotes(groupKey.getValueDisplayName()), @@ -1694,7 +1694,7 @@ public final class DrawableDB { tskCase.getCaseDbAccessManager().select(groupAnalyzedQueryStmt, queryResultProcessor); return queryResultProcessor.getIsAnalyzed(); } catch ( TskCoreException ex) { - String msg = String.format("Failed to get group isAnalyzed for group key %s", groupKey.getValueDisplayName()); //NON-NLS + String msg = String.format("Failed to get group is_analyzed for group key %s", groupKey.getValueDisplayName()); //NON-NLS logger.log(Level.SEVERE, msg, ex); } @@ -1889,7 +1889,7 @@ public final class DrawableDB { } int isAnalyzed = (groupBy == DrawableAttribute.PATH) ? 0 : 1; - String insertSQL = String.format(" (data_source_obj_id, value, attribute, isAnalyzed) VALUES (%d, \'%s\', \'%s\', %d)", + String insertSQL = String.format(" (data_source_obj_id, value, attribute, is_analyzed) VALUES (%d, \'%s\', \'%s\', %d)", ds_obj_id, SleuthkitCase.escapeSingleQuotes(value), SleuthkitCase.escapeSingleQuotes(groupBy.attrName.toString()), isAnalyzed); if (DbType.POSTGRESQL == tskCase.getDatabaseType()) { insertSQL += " ON CONFLICT DO NOTHING"; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 12fde98654..c0c87c821f 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -108,11 +108,11 @@ public class GroupManager { private final ImageGalleryController controller; /** - * Keeps track of the last path group - * - a change in path indicates the last path group is analyzed + * Keeps track of the current path group + * - a change in path indicates the current path group is analyzed */ @GuardedBy("this") //NOPMD - private GroupKey lastUpdatedPathGroup = null; + private GroupKey currentPathGroup = null; /** * list of all analyzed groups */ @@ -625,7 +625,7 @@ public class GroupManager { // see if a group has been created yet for the key DrawableGroup g = getGroupForKey(gk); - checkForPathGroupChange(gk); + updateCurrentPathGroup(gk); addFileToGroup(g, gk, fileId); } } @@ -635,7 +635,9 @@ public class GroupManager { } /** - * Checks if the given path is different from the last updated path group. + * Checks if the given path is different from the current path group. + * If so, updates the current path group as analyzed, and sets current path + * group to the given path. * * The idea is that when the path of the files being processed changes, * we have moved from one folder to the next, and the group for the @@ -647,40 +649,40 @@ public class GroupManager { * * @param groupKey */ - private void checkForPathGroupChange(GroupKey groupKey) { + synchronized private void updateCurrentPathGroup(GroupKey groupKey) { try { if (groupKey.getAttribute() == DrawableAttribute.PATH) { - if (this.lastUpdatedPathGroup == null) { - lastUpdatedPathGroup = groupKey; + if (this.currentPathGroup == null) { + currentPathGroup = groupKey; } - else if (groupKey.getValue().toString().equalsIgnoreCase(this.lastUpdatedPathGroup.getValue().toString()) == false) { + else if (groupKey.getValue().toString().equalsIgnoreCase(this.currentPathGroup.getValue().toString()) == false) { // mark the last path group as analyzed - getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup); - popuplateIfAnalyzed(lastUpdatedPathGroup, null); + getDrawableDB().markGroupAnalyzed(currentPathGroup); + popuplateIfAnalyzed(currentPathGroup, null); - lastUpdatedPathGroup = groupKey; + currentPathGroup = groupKey; } } } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Error setting isAnalyzed status for group: %s", groupKey.getValue().toString()), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error setting is_analyzed status for group: %s", groupKey.getValue().toString()), ex); //NON-NLS } } /** - * Resets the last updated path group, after marking the last path group as analyzed. + * Resets current path group, after marking the current path group as analyzed. */ - public void resetLastUpdatedPathGroup() { + synchronized public void resetCurrentPathGroup() { try { - if (lastUpdatedPathGroup != null) { - getDrawableDB().markGroupAnalyzed(lastUpdatedPathGroup); - popuplateIfAnalyzed(lastUpdatedPathGroup, null); - lastUpdatedPathGroup = null; + if (currentPathGroup != null) { + getDrawableDB().markGroupAnalyzed(currentPathGroup); + popuplateIfAnalyzed(currentPathGroup, null); + currentPathGroup = null; } } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Error resetting last path group: %s", lastUpdatedPathGroup.getValue().toString()), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error resetting last path group: %s", currentPathGroup.getValue().toString()), ex); //NON-NLS } } /**