codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-10-05 08:46:20 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4037 from raman-bt/1009-ig-rebuild-by-datasource

Browse Source 
1009: Rebuild IG at data-source level
This commit is contained in:
Richard Cordovano
2018-08-21 15:35:28 -04:00
committed by GitHub
parent d1a900a08f ae5621db16
commit 0ef200f9f7
4 changed files with 121 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java
+91 -79
View File
@@ -25,6 +25,7 @@ import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Executors;
@@ -74,6 +75,7 @@ import org.sleuthkit.autopsy.events.AutopsyEvent;
import org.sleuthkit.autopsy.imagegallery.actions.UndoRedoManager;
import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableFile;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableTagsManager;
import org.sleuthkit.autopsy.imagegallery.datamodel.HashSetManager;
@@ -200,11 +202,6 @@ public final class ImageGalleryController {
Platform.runLater(() -> {
stale.set(b);
});
try {
new PerCaseProperties(Case.getCurrentCaseThrows()).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.STALE, b.toString());
} catch (NoCurrentCaseException ex) {
Logger.getLogger(ImageGalleryController.class.getName()).log(Level.WARNING, "Exception while getting open case."); //NON-NLS
}
}
public ReadOnlyBooleanProperty stale() {
@@ -403,7 +400,10 @@ public final class ImageGalleryController {
*
*/
public void rebuildDB() {
queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase));
// queue a rebuild task for each stale data source
getStaleDataSourceIds().forEach((dataSourceObjId) -> {
queueDBTask(new CopyAnalyzedFiles(dataSourceObjId, instance, db, sleuthKitCase));
});
}
/**
@@ -437,48 +437,57 @@ public final class ImageGalleryController {
* @return true if datasources table is stale
*/
boolean isDataSourcesTableStale() {
return (getStaleDataSourceIds().isEmpty() == false);
}
/**
* Returns a set of data source object ids that are stale.
*
* This includes any data sources already in the table, that are not in COMPLETE status,
* or any data sources that might have been added to the case, but are not in the datasources table.
*
* @return list of data source object ids that are stale.
*/
Set<Long> getStaleDataSourceIds() {
Set<Long> staleDataSourceIds = new HashSet<>();
// no current case open to check
if ((null == getDatabase()) || (null == getSleuthKitCase())) {
return false;
return staleDataSourceIds;
}
try {
Set<Long> knownDataSourceIds= getDatabase().getDataSourceIds();
Map<Long, DrawableDbBuildStatusEnum> knownDataSourceIds= getDatabase().getDataSourceDbBuildStatus();
List<DataSource> dataSources = getSleuthKitCase().getDataSources();
Set<Long> caseDataSourceIds = new HashSet<>();
dataSources.forEach((dataSource) -> {
caseDataSourceIds.add(dataSource.getId());
});
return !(knownDataSourceIds.containsAll(caseDataSourceIds) && caseDataSourceIds.containsAll(knownDataSourceIds));
// collect all data sources already in the table, that are not yet COMPLETE
knownDataSourceIds.entrySet().stream().forEach((Map.Entry<Long, DrawableDbBuildStatusEnum> t) -> {
DrawableDbBuildStatusEnum status = t.getValue();
if (DrawableDbBuildStatusEnum.COMPLETE != status) {
staleDataSourceIds.add(t.getKey());
}
});
// collect any new data sources in the case.
caseDataSourceIds.forEach((Long id) -> {
if (!knownDataSourceIds.containsKey(id)) {
staleDataSourceIds.add(id);
}
});
return staleDataSourceIds;
}
catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Image Gallery failed to check if datasources table is stale.", ex);
return false;
}
}
/**
* Update the datasources table in drawable DB.
*
*/
private void updateDataSourcesTable() {
// no current case open to update
if ((null == getDatabase()) || (null == getSleuthKitCase())) {
return;
return staleDataSourceIds;
}
try {
List<DataSource> dataSources = getSleuthKitCase().getDataSources();
dataSources.forEach((dataSource) -> {
getDatabase().insertDataSource(dataSource.getId());
});
}
catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Image Gallery failed to update data_sources table.", ex);
}
}
synchronized private void shutDownDBExecutor() {
@@ -727,7 +736,10 @@ public final class ImageGalleryController {
@NbBundle.Messages({"BulkTask.committingDb.status=committing image/video database",
"BulkTask.stopCopy.status=Stopping copy to drawable db task.",
"BulkTask.errPopulating.errMsg=There was an error populating Image Gallery database."})
/* Base abstract class for various methods of copying data into the Image gallery DB */
/**
* Base abstract class for various methods of copying image files data,
* for a given data source, into the Image gallery DB.
*/
abstract static private class BulkTransferTask extends BackgroundTask {
static private final String FILE_EXTENSION_CLAUSE =
@@ -739,27 +751,35 @@ public final class ImageGalleryController {
"(mime_type LIKE '" //NON-NLS
+ String.join("' OR mime_type LIKE '", FileTypeUtils.getAllSupportedMimeTypes()) //NON-NLS
+ "') ";
static final String DRAWABLE_QUERY =
//grab files with supported extension
"(" + FILE_EXTENSION_CLAUSE
//grab files with supported mime-types
+ " OR " + MIMETYPE_CLAUSE //NON-NLS
//grab files with image or video mime-types even if we don't officially support them
+ " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )"; //NON-NLS
final String DRAWABLE_QUERY;
final String DATASOURCE_CLAUSE;
final ImageGalleryController controller;
final DrawableDB taskDB;
final SleuthkitCase tskCase;
final long dataSourceObjId;
ProgressHandle progressHandle;
private boolean taskCompletionStatus;
BulkTransferTask(ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
BulkTransferTask(long dataSourceObjId, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
this.controller = controller;
this.taskDB = taskDB;
this.tskCase = tskCase;
this.dataSourceObjId = dataSourceObjId;
DATASOURCE_CLAUSE = " (data_source_obj_id = " + dataSourceObjId + ") ";
DRAWABLE_QUERY =
DATASOURCE_CLAUSE +
" AND ( " +
//grab files with supported extension
FILE_EXTENSION_CLAUSE
//grab files with supported mime-types
+ " OR " + MIMETYPE_CLAUSE //NON-NLS
//grab files with image or video mime-types even if we don't officially support them
+ " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )"; //NON-NLS
}
/**
@@ -768,8 +788,16 @@ public final class ImageGalleryController {
*/
abstract void cleanup(boolean success);
abstract List<AbstractFile> getFiles() throws TskCoreException;
/**
* Gets a list of files to process.
*
* @return list of files to process
* @throws TskCoreException
*/
List<AbstractFile> getFiles() throws TskCoreException {
return tskCase.findAllFilesWhere(DRAWABLE_QUERY);
}
abstract void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDBTransaction) throws TskCoreException;
@Override
@@ -786,8 +814,9 @@ public final class ImageGalleryController {
final List<AbstractFile> files = getFiles();
progressHandle.switchToDeterminate(files.size());
updateProgress(0.0);
taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS);
updateProgress(0.0);
taskCompletionStatus = true;
int workDone = 0;
@@ -839,6 +868,9 @@ public final class ImageGalleryController {
return;
} finally {
progressHandle.finish();
if (taskCompletionStatus) {
taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.COMPLETE);
}
updateMessage("");
updateProgress(-1.0);
}
@@ -862,23 +894,20 @@ public final class ImageGalleryController {
@NbBundle.Messages({"CopyAnalyzedFiles.committingDb.status=committing image/video database",
"CopyAnalyzedFiles.stopCopy.status=Stopping copy to drawable db task.",
"CopyAnalyzedFiles.errPopulating.errMsg=There was an error populating Image Gallery database."})
static private class CopyAnalyzedFiles extends BulkTransferTask {
private class CopyAnalyzedFiles extends BulkTransferTask {
CopyAnalyzedFiles(ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
super(controller, taskDB, tskCase);
CopyAnalyzedFiles(long dataSourceObjId, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
super(dataSourceObjId, controller, taskDB, tskCase);
}
@Override
protected void cleanup(boolean success) {
// processFile will set success to fail if files are missing MIME types
controller.setStale(!success);
// at the end of the task, set the stale status based on the
// cumulative status of all data sources
controller.setStale(isDataSourcesTableStale());
}
@Override
List<AbstractFile> getFiles() throws TskCoreException {
controller.updateDataSourcesTable();
return tskCase.findAllFilesWhere(DRAWABLE_QUERY);
}
@Override
void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDbTransaction) throws TskCoreException {
@@ -928,16 +957,13 @@ public final class ImageGalleryController {
static private class PrePopulateDataSourceFiles extends BulkTransferTask {
private static final Logger LOGGER = Logger.getLogger(PrePopulateDataSourceFiles.class.getName());
private final Content dataSource;
/**
*
* @param dataSourceId Data source object ID
*/
PrePopulateDataSourceFiles(Content dataSource, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
super(controller, taskDB, tskCase);
this.dataSource = dataSource;
PrePopulateDataSourceFiles(long dataSourceObjId, ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
super(dataSourceObjId, controller, taskDB, tskCase);
}
@Override
@@ -949,13 +975,6 @@ public final class ImageGalleryController {
taskDB.insertFile(DrawableFile.create(f, false, false), tr, caseDBTransaction);
}
@Override
List<AbstractFile> getFiles() throws TskCoreException {
long datasourceID = dataSource.getDataSource().getId();
taskDB.insertDataSource(datasourceID);
return tskCase.findAllFilesWhere("data_source_obj_id = " + datasourceID + " AND " + DRAWABLE_QUERY);
}
@Override
@NbBundle.Messages({"PrePopulateDataSourceFiles.prepopulatingDb.status=prepopulating image/video database",})
ProgressHandle getInitialProgressHandle() {
@@ -1020,9 +1039,7 @@ public final class ImageGalleryController {
"Unable to determine if file is drawable and not known. Not making any changes to DB. See the logs for details.");
}
}
} else {
setStale(true);
}
}
}
break;
}
@@ -1058,13 +1075,8 @@ public final class ImageGalleryController {
if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) {
Content newDataSource = (Content) evt.getNewValue();
if (isListeningEnabled()) {
queueDBTask(new PrePopulateDataSourceFiles(newDataSource, ImageGalleryController.this, getDatabase(), getSleuthKitCase()));
} else {
setStale(true);
}
}
else {
setStale(true);
queueDBTask(new PrePopulateDataSourceFiles(newDataSource.getId(), ImageGalleryController.this, getDatabase(), getSleuthKitCase()));
}
}
break;
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java
+1 -5
View File
@@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.imagegallery;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.commons.lang3.StringUtils;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
@@ -85,10 +84,7 @@ public class ImageGalleryModule {
*/
public static boolean isDrawableDBStale(Case c) {
if (c != null) {
String stale = new PerCaseProperties(c).getConfigSetting(ImageGalleryModule.MODULE_NAME, PerCaseProperties.STALE);
return ( ImageGalleryController.getDefault().isDataSourcesTableStale() ||
(StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : false) );
return ImageGalleryController.getDefault().isDataSourcesTableStale();
} else {
return false;
}
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/PerCaseProperties.java
-2
View File
@@ -40,8 +40,6 @@ class PerCaseProperties {
public static final String ENABLED = "enabled"; //NON-NLS
public static final String STALE = "stale"; //NON-NLS
private final Case theCase;
PerCaseProperties(Case c) {
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java
+29 -18
View File
@@ -102,7 +102,7 @@ public final class DrawableDB {
private final PreparedStatement insertHashHitStmt;
private final PreparedStatement insertDataSourceStmt;
private final PreparedStatement updateDataSourceStmt;
private final PreparedStatement updateFileStmt;
private final PreparedStatement insertFileStmt;
@@ -149,6 +149,15 @@ public final class DrawableDB {
private final SleuthkitCase tskCase;
private final ImageGalleryController controller;
/**
* Enum to track Image gallery db rebuild status for a data source
*/
public enum DrawableDbBuildStatusEnum {
UNKNOWN, /// no known status
IN_PROGRESS, /// drawable db rebuild has been started for the data source
COMPLETE; /// drawable db rebuild is complete for the data source
}
//////////////general database logic , mostly borrowed from sleuthkitcase
/**
* Lock to protect against concurrent write accesses to case database and to
@@ -210,9 +219,9 @@ public final class DrawableDB {
"INSERT OR IGNORE INTO drawable_files (obj_id , path, name, created_time, modified_time, make, model, analyzed) " //NON-NLS
+ "VALUES (?,?,?,?,?,?,?,?)"); //NON-NLS
insertDataSourceStmt = prepareStatement(
"INSERT OR IGNORE INTO datasources (ds_obj_id) " //NON-NLS
+ "VALUES (?)"); //NON-NLS
updateDataSourceStmt = prepareStatement(
"INSERT OR REPLACE INTO datasources (ds_obj_id, drawable_db_build_status) " //NON-NLS
+ " VALUES (?,?)"); //NON-NLS
removeFileStmt = prepareStatement("DELETE FROM drawable_files WHERE obj_id = ?"); //NON-NLS
@@ -374,7 +383,8 @@ public final class DrawableDB {
try (Statement stmt = con.createStatement()) {
String sql = "CREATE TABLE if not exists datasources " //NON-NLS
+ "( id INTEGER PRIMARY KEY, " //NON-NLS
+ " ds_obj_id integer UNIQUE NOT NULL)"; //NON-NLS
+ " ds_obj_id integer UNIQUE NOT NULL, "
+ " drawable_db_build_status VARCHAR(128) )"; //NON-NLS
stmt.execute(sql);
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "problem creating datasources table", ex); //NON-NLS
@@ -763,20 +773,20 @@ public final class DrawableDB {
/**
* Gets all data source object ids from datasources table
* Gets all data source object ids from datasources table, and their DrawableDbBuildStatusEnum
*
* @return list of known data source object ids
* @return map of known data source object ids, and their db status
*/
public Set<Long> getDataSourceIds() throws TskCoreException {
public Map<Long, DrawableDbBuildStatusEnum> getDataSourceDbBuildStatus() throws TskCoreException {
Statement statement = null;
ResultSet rs = null;
Set<Long> ret = new HashSet<>();
Map<Long, DrawableDbBuildStatusEnum> map = new HashMap<>();
dbReadLock();
try {
statement = con.createStatement();
rs = statement.executeQuery("SELECT ds_obj_id FROM datasources "); //NON-NLS
rs = statement.executeQuery("SELECT ds_obj_id, drawable_db_build_status FROM datasources "); //NON-NLS
while (rs.next()) {
ret.add(rs.getLong(1));
map.put(rs.getLong("ds_obj_id"), DrawableDbBuildStatusEnum.valueOf(rs.getString("drawable_db_build_status")));
}
} catch (SQLException e) {
throw new TskCoreException("SQLException while getting data source object ids", e);
@@ -797,24 +807,25 @@ public final class DrawableDB {
}
dbReadUnlock();
}
return ret;
return map;
}
/**
* Insert given data source object id into datasources table
* Insert/update given data source object id and it's DB rebuild status in the datasources table.
*
* If the object id exists in the table already, it does nothing.
* If the object id exists in the table already, it updates the status
*
* @param dsObjectId data source object id to insert
*/
public void insertDataSource(long dsObjectId) {
public void insertOrUpdateDataSource(long dsObjectId, DrawableDbBuildStatusEnum status ) {
dbWriteLock();
try {
// "INSERT OR IGNORE/ INTO datasources (ds_obj_id)"
insertDataSourceStmt.setLong(1,dsObjectId);
// "INSERT OR REPLACE INTO datasources (ds_obj_id, drawable_db_build_status) " //NON-NLS
updateDataSourceStmt.setLong(1,dsObjectId);
updateDataSourceStmt.setString(2, status.name());
insertDataSourceStmt.executeUpdate();
updateDataSourceStmt.executeUpdate();
} catch (SQLException | NullPointerException ex) {
LOGGER.log(Level.SEVERE, "failed to insert/update datasources table", ex); //NON-NLS
} finally {