From 0906924287e684d8c5cade27a45b87cafe4a69bf Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 9 Jul 2014 13:01:16 -0400 Subject: [PATCH 01/16] Update FileManager for new SleuthkitCase transaction API --- .../autopsy/casemodule/services/FileManager.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java index 5b83e2d9b8..2ab817144d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.VirtualDirectoryNode; @@ -36,13 +35,14 @@ import org.sleuthkit.autopsy.ingest.ModuleContentEvent; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DerivedFile; -import org.sleuthkit.datamodel.LocalFile; -import org.sleuthkit.datamodel.VirtualDirectory; import org.sleuthkit.datamodel.LayoutFile; +import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; import org.sleuthkit.datamodel.Transaction; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskFileRange; +import org.sleuthkit.datamodel.VirtualDirectory; /** * Abstraction to facilitate access to files and directories. @@ -267,7 +267,7 @@ public class FileManager implements Closeable { rootsToAdd.add(localFile); } - Transaction trans = tskCase.createTransaction(); + CaseDbTransaction trans = tskCase.beginTransaction(); // make a virtual top-level directory for this set of files/dirs final VirtualDirectory fileSetRootDir = addLocalFileSetRootDir(trans); @@ -295,8 +295,6 @@ public class FileManager implements Closeable { trans.commit(); } catch (TskCoreException ex) { trans.rollback(); - } finally { - trans.close(); } return fileSetRootDir; } @@ -309,7 +307,7 @@ public class FileManager implements Closeable { * * @throws TskCoreException */ - private VirtualDirectory addLocalFileSetRootDir(Transaction trans) throws TskCoreException { + private VirtualDirectory addLocalFileSetRootDir(CaseDbTransaction trans) throws TskCoreException { VirtualDirectory created = null; @@ -343,7 +341,7 @@ public class FileManager implements Closeable { * directory. * @throws TskCoreException */ - private AbstractFile addLocalDirInt(Transaction trans, VirtualDirectory parentVd, + private AbstractFile addLocalDirInt(CaseDbTransaction trans, VirtualDirectory parentVd, java.io.File localFile, FileAddProgressUpdater addProgressUpdater) throws TskCoreException { if (tskCase == null) { @@ -399,7 +397,7 @@ public class FileManager implements Closeable { * due to a critical system error or of the file * manager has already been closed */ - private synchronized LocalFile addLocalFileInt(AbstractFile parentFile, java.io.File localFile, Transaction trans) throws TskCoreException { + private synchronized LocalFile addLocalFileInt(AbstractFile parentFile, java.io.File localFile, CaseDbTransaction trans) throws TskCoreException { if (tskCase == null) { throw new TskCoreException( From e5230e5a79c1d00c7d9f4a4279b6c9bf5cefc77d Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Mon, 28 Jul 2014 13:52:38 -0400 Subject: [PATCH 02/16] Reduce database round trips in registry parsing code --- .../recentactivity/ExtractRegistry.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index a133d1618b..0a55c1e42f 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -191,7 +191,7 @@ class ExtractRegistry extends Extract { // parse the autopsy-specific output if (regOutputFiles.autopsyPlugins.isEmpty() == false) { - if (parseAutopsyPluginOutput(regOutputFiles.autopsyPlugins, regFile.getId(), usbMapper) == false) { + if (parseAutopsyPluginOutput(regOutputFiles.autopsyPlugins, regFile, usbMapper) == false) { this.addErrorMessage( NbBundle.getMessage(this.getClass(), "ExtractRegistry.analyzeRegFiles.failedParsingResults", this.getName(), regFileName)); @@ -371,7 +371,14 @@ class ExtractRegistry extends Extract { } // @@@ VERIFY that we are doing the right thing when we parse multiple NTUSER.DAT - private boolean parseAutopsyPluginOutput(String regRecord, long orgId, UsbDeviceIdMapper extrctr) { + /** + * + * @param regRecord + * @param regFile File object for registry that we are parsing (to make blackboard artifacts with) + * @param extrctr + * @return + */ + private boolean parseAutopsyPluginOutput(String regRecord, AbstractFile regFile, UsbDeviceIdMapper extrctr) { FileInputStream fstream = null; try { SleuthkitCase tempDb = currentCase.getSleuthkitCase(); @@ -447,7 +454,7 @@ class ExtractRegistry extends Extract { Long usbMtime = Long.parseLong(artnode.getAttribute("mtime")); //NON-NLS usbMtime = Long.valueOf(usbMtime.toString()); - BlackboardArtifact bbart = tempDb.getContentById(orgId).newArtifact(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED); + BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace"), usbMtime)); @@ -494,7 +501,7 @@ class ExtractRegistry extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace"), itemMtime)); - BlackboardArtifact bbart = tempDb.getContentById(orgId).newArtifact(ARTIFACT_TYPE.TSK_INSTALLED_PROG); + BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_INSTALLED_PROG); bbart.addAttributes(bbattributes); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding installed program artifact to blackboard."); //NON-NLS @@ -526,7 +533,7 @@ class ExtractRegistry extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace"), installtime)); - BlackboardArtifact bbart = tempDb.getContentById(orgId).newArtifact(ARTIFACT_TYPE.TSK_INSTALLED_PROG); + BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_INSTALLED_PROG); bbart.addAttributes(bbattributes); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding installed program artifact to blackboard."); //NON-NLS @@ -537,7 +544,7 @@ class ExtractRegistry extends Extract { String name = artnode.getAttribute("name"); //NON-NLS try { - BlackboardArtifact bbart = tempDb.getContentById(orgId).newArtifact(ARTIFACT_TYPE.TSK_RECENT_OBJECT); + BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_RECENT_OBJECT); // @@@ BC: Consider removing this after some more testing. It looks like an Mtime associated with the root key and not the individual item if (mtime != null) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(), From 56bc933c61054e68b171272310ede42cb32520a4 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Mon, 28 Jul 2014 14:13:33 -0400 Subject: [PATCH 03/16] Added hasData method to Case to reduce DB trips --- .../sleuthkit/autopsy/casemodule/Case.java | 28 ++++++++++++++++--- .../DataContentTopComponent.java | 2 +- .../corecomponents/DataResultPanel.java | 2 +- .../DataResultTopComponent.java | 2 +- .../DirectoryTreeTopComponent.java | 4 +-- .../menuactions/DataContentDynamicMenu.java | 2 +- .../menuactions/DataExplorerDynamicMenu.java | 2 +- .../sleuthkit/autopsy/timeline/Timeline.java | 2 +- 8 files changed, 32 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 7a50c3efb0..44ae49269c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -140,6 +140,9 @@ public class Case implements SleuthkitCase.ErrorObserver { private static final Logger logger = Logger.getLogger(Case.class.getName()); static final String CASE_EXTENSION = "aut"; //NON-NLS static final String CASE_DOT_EXTENSION = "." + CASE_EXTENSION; + + // we cache if the case has data in it yet since a few places ask for it and we dont' need to keep going to DB + private boolean hasData = false; /** * Constructor for the Case class @@ -795,11 +798,14 @@ public class Case implements SleuthkitCase.ErrorObserver { */ public Long[] getImageIDs() { Set ids = getImagePaths(db).keySet(); + hasData = (ids.size() > 0); return ids.toArray(new Long[ids.size()]); } public List getImages() throws TskCoreException { - return db.getImages(); + List list = db.getImages(); + hasData = (list.size() > 0); + return list; } /** @@ -818,7 +824,9 @@ public class Case implements SleuthkitCase.ErrorObserver { */ public List getRootObjects() { try { - return db.getRootObjects(); + List list = db.getRootObjects(); + hasData = (list.size() > 0); + return list; } catch (TskException ex) { throw new RuntimeException(NbBundle.getMessage(this.getClass(), "Case.exception.errGetRootObj"), ex); } @@ -1113,7 +1121,7 @@ public class Case implements SleuthkitCase.ErrorObserver { CallableSystemAction.get(CasePropertiesAction.class).setEnabled(true); CallableSystemAction.get(CaseDeleteAction.class).setEnabled(true); // Delete Case menu - if (toChangeTo.getRootObjectsCount() > 0) { + if (toChangeTo.hasData()) { // open all top components CoreComponentControl.openCoreWindows(); } else { @@ -1160,7 +1168,7 @@ public class Case implements SleuthkitCase.ErrorObserver { //delete image helper private void doDeleteImage() { // no more image left in this case - if (currentCase.getRootObjectsCount() == 0) { + if (currentCase.hasData()) { // close all top components CoreComponentControl.closeCoreWindows(); } @@ -1193,4 +1201,16 @@ public class Case implements SleuthkitCase.ErrorObserver { public List getAllReports() throws TskCoreException { return this.db.getAllReports(); } + + /** + * Returns if the case has data in it yet. + * @return + */ + public boolean hasData() { + // false is also the initial value, so make the DB trip if it is still false + if (!hasData) { + hasData = (getRootObjectsCount() > 0); + } + return hasData; + } } diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java index a972c4bf0d..5a65cf4b62 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java @@ -166,7 +166,7 @@ public final class DataContentTopComponent extends TopComponent implements DataC @Override public boolean canClose() { - return (!this.isDefault) || !Case.existsCurrentCase() || Case.getCurrentCase().getRootObjectsCount() == 0; // only allow this window to be closed when there's no case opened or no image in this case + return (!this.isDefault) || !Case.existsCurrentCase() || Case.getCurrentCase().hasData() == false; // only allow this window to be closed when there's no case opened or no image in this case } @Override diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java index 0a897a61c2..88d8706e71 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java @@ -457,7 +457,7 @@ public class DataResultPanel extends javax.swing.JPanel implements DataResult, C } public boolean canClose() { - return (!this.isMain) || !Case.existsCurrentCase() || Case.getCurrentCase().getRootObjectsCount() == 0; // only allow this window to be closed when there's no case opened or no image in this case + return (!this.isMain) || !Case.existsCurrentCase() || Case.getCurrentCase().hasData() == false; // only allow this window to be closed when there's no case opened or no image in this case } @Override diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultTopComponent.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultTopComponent.java index 0f844dfa2d..e11c13586e 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultTopComponent.java @@ -298,7 +298,7 @@ public class DataResultTopComponent extends TopComponent implements DataResult, @Override public boolean canClose() { - return (!this.isMain) || !Case.existsCurrentCase() || Case.getCurrentCase().getRootObjectsCount() == 0; // only allow this window to be closed when there's no case opened or no image in this case + return (!this.isMain) || !Case.existsCurrentCase() || Case.getCurrentCase().hasData() == false; // only allow this window to be closed when there's no case opened or no image in this case } /** diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 785f9952df..06a733046e 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -349,7 +349,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat Case currentCase = Case.getCurrentCase(); // close the top component if there's no image in this case - if (currentCase.getRootObjectsCount() == 0) { + if (currentCase.hasData() == false) { //this.close(); ((BeanTreeView) this.jScrollPane1).setRootVisible(false); // hide the root } else { @@ -483,7 +483,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat @Override public boolean canClose() { - return !Case.existsCurrentCase() || Case.getCurrentCase().getRootObjectsCount() == 0; // only allow this window to be closed when there's no case opened or no image in this case + return !Case.existsCurrentCase() || Case.getCurrentCase().hasData() == false; // only allow this window to be closed when there's no case opened or no image in this case } /** diff --git a/Core/src/org/sleuthkit/autopsy/menuactions/DataContentDynamicMenu.java b/Core/src/org/sleuthkit/autopsy/menuactions/DataContentDynamicMenu.java index 75a5e2533b..a338eb099d 100644 --- a/Core/src/org/sleuthkit/autopsy/menuactions/DataContentDynamicMenu.java +++ b/Core/src/org/sleuthkit/autopsy/menuactions/DataContentDynamicMenu.java @@ -51,7 +51,7 @@ import org.sleuthkit.autopsy.corecomponents.DataContentTopComponent; defaultItem.addActionListener(new OpenTopComponentAction(contentWin)); - if (!Case.existsCurrentCase() || Case.getCurrentCase().getRootObjectsCount() == 0) { + if (!Case.existsCurrentCase() || Case.getCurrentCase().hasData() == false) { defaultItem.setEnabled(false); // disable the menu items when no case is opened } else { defaultItem.setEnabled(true); // enable the menu items when there's a case opened / created diff --git a/Core/src/org/sleuthkit/autopsy/menuactions/DataExplorerDynamicMenu.java b/Core/src/org/sleuthkit/autopsy/menuactions/DataExplorerDynamicMenu.java index 3ed7e67b76..1bbe31412b 100644 --- a/Core/src/org/sleuthkit/autopsy/menuactions/DataExplorerDynamicMenu.java +++ b/Core/src/org/sleuthkit/autopsy/menuactions/DataExplorerDynamicMenu.java @@ -53,7 +53,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer; JMenuItem item = new JMenuItem(explorerWin.getName()); item.addActionListener(new OpenTopComponentAction(explorerWin)); - if(!Case.existsCurrentCase() || Case.getCurrentCase().getRootObjectsCount() == 0){ + if(!Case.existsCurrentCase() || Case.getCurrentCase().hasData() == false){ item.setEnabled(false); // disable the menu when no case is opened } else{ diff --git a/Core/src/org/sleuthkit/autopsy/timeline/Timeline.java b/Core/src/org/sleuthkit/autopsy/timeline/Timeline.java index 25cc1d16f6..0e8cb21110 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/Timeline.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/Timeline.java @@ -1161,7 +1161,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, skCase = currentCase.getSleuthkitCase(); try { - if (currentCase.getRootObjectsCount() == 0) { + if (currentCase.hasData() == false) { logger.log(Level.INFO, "Error creating timeline, there are no data sources. "); //NON-NLS } else { From 7ffe07c00c8273974dd16049b799c574de9df309 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 29 Jul 2014 15:24:00 -0400 Subject: [PATCH 04/16] Remove thumbnail supported test from DataResultViewerThumbnail.isSupported() --- .../corecomponents/DataResultViewerThumbnail.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerThumbnail.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerThumbnail.java index 232505319a..37468a169d 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerThumbnail.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerThumbnail.java @@ -299,14 +299,7 @@ import org.sleuthkit.datamodel.TskCoreException; if (selectedNode == null) { return false; } - - Children ch = selectedNode.getChildren(); - for (Node n : ch.getNodes()) { - if (ThumbnailViewChildren.isSupported(n)) { - return true; - } - } - return false; + return true; } @Override From 68ead3dc85e2c4ba17428db65ec5a79bc43f1244 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 29 Jul 2014 15:41:47 -0400 Subject: [PATCH 05/16] Clarify javadocs comment for ReportModule.getRelativeFilePath --- .../sleuthkit/autopsy/report/ReportModule.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportModule.java b/Core/src/org/sleuthkit/autopsy/report/ReportModule.java index 288d53acf6..2d0ab6b0d2 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportModule.java @@ -25,7 +25,7 @@ package org.sleuthkit.autopsy.report; /** * Interface for report modules that plug in to the reporting infrastructure. */ - interface ReportModule { +interface ReportModule { /** * Get the name of the report this module generates. @@ -37,13 +37,14 @@ package org.sleuthkit.autopsy.report; * module generates. */ public String getDescription(); - + /** - * Gets the path of the report file, if any, generated by this module. If - * a report file is generated, the path should be relative to the reports - * directory. - * - * @return File path relative to the reports directory, may be null. + * Gets the relative path of the report file, if any, generated by this + * module. The path should be relative to the time stamp subdirectory of + * reports directory. + * + * @return Report file path relative to the time stamp subdirectory reports + * directory, may be null if the module does not produce a report file. */ public String getRelativeFilePath(); -} \ No newline at end of file +} From 58385478f11399b5eac6fce64d855400fef5a6ab Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 30 Jul 2014 15:17:10 -0400 Subject: [PATCH 06/16] Remove unused import from FileManager.java --- .../org/sleuthkit/autopsy/casemodule/services/FileManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java index 2ab817144d..bf3db27add 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java @@ -39,7 +39,6 @@ import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; -import org.sleuthkit.datamodel.Transaction; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskFileRange; import org.sleuthkit.datamodel.VirtualDirectory; From f8fe5694ef12090d50ec38b963503f178bd85a8d Mon Sep 17 00:00:00 2001 From: jmillman Date: Wed, 30 Jul 2014 17:08:21 -0400 Subject: [PATCH 07/16] pass AbstractFile as part of FILE_DONE event --- .../autopsy/ingest/FileIngestPipeline.java | 4 +--- .../autopsy/ingest/IngestManager.java | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index c4dbef3917..573cf96a06 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -19,9 +19,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.sleuthkit.datamodel.AbstractFile; /** @@ -112,7 +110,7 @@ final class FileIngestPipeline { } file.close(); if (!context.isJobCancelled()) { - IngestManager.getInstance().fireFileIngestDone(file.getId()); + IngestManager.getInstance().fireFileIngestDone(file); } ingestManager.setIngestTaskProgressCompleted(task); return errors; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 944a482471..008bf410db 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -31,17 +31,17 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; +import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandleFactory; import org.openide.util.Cancellable; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.datamodel.Content; -import javax.swing.JOptionPane; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.core.UserPreferences; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; /** * Manages the execution of ingest jobs. @@ -288,7 +288,7 @@ public class IngestManager { /** * Property change event fired when the ingest of a file is completed. * The old value of the PropertyChangeEvent is the Autopsy object ID of - * the file, and the new value is set to null. + * the file. The new value is null or an AbstractFile for that ID. */ FILE_DONE, }; @@ -389,6 +389,15 @@ public class IngestManager { fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, fileId, null)); } + /** + * Fire an ingest event signifying the ingest of a file is completed. + * + * @param file The file that is completed. + */ + void fireFileIngestDone(AbstractFile file) { + fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); + } + /** * Fire an event signifying a blackboard post by an ingest module. * From 8cbbfe79b67635cc73f3a3bcc7d5bfb68d4cb86f Mon Sep 17 00:00:00 2001 From: jmillman Date: Wed, 30 Jul 2014 17:39:22 -0400 Subject: [PATCH 08/16] removed unused method fireFileIngestDone(long fileId) --- .../org/sleuthkit/autopsy/ingest/IngestManager.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 008bf410db..3c1e23cb83 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -288,7 +288,7 @@ public class IngestManager { /** * Property change event fired when the ingest of a file is completed. * The old value of the PropertyChangeEvent is the Autopsy object ID of - * the file. The new value is null or an AbstractFile for that ID. + * the file. The new value is the AbstractFile for that ID. */ FILE_DONE, }; @@ -380,15 +380,6 @@ public class IngestManager { fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); } - /** - * Fire an ingest event signifying the ingest of a file is completed. - * - * @param fileId The object id of file. - */ - void fireFileIngestDone(long fileId) { - fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, fileId, null)); - } - /** * Fire an ingest event signifying the ingest of a file is completed. * From 7298add0e3e8d46b91573e7f10ad6e1830a2e310 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 4 Aug 2014 17:12:23 -0400 Subject: [PATCH 09/16] Update AddImageTask for non-static case db locking API --- .../autopsy/casemodule/AddImageTask.java | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java index affa9fdb1a..3c3e647cb4 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java @@ -135,39 +135,29 @@ import org.sleuthkit.datamodel.TskException; * @throws Exception */ @Override - public void run() { - - errorList.clear(); - - //lock DB for writes in this thread - SleuthkitCase.acquireExclusiveLock(); - - addImageProcess = currentCase.makeAddImageProcess(timeZone, true, noFatOrphans); - dirFetcher = new Thread( new CurrentDirectoryFetcher(progressMonitor, addImageProcess)); - + public void run() { + errorList.clear(); try { - progressMonitor.setIndeterminate(true); - progressMonitor.setProgress(0); - - dirFetcher.start(); - - addImageProcess.run(new String[]{this.imagePath}); - - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Core errors occurred while running add image. ", ex); //NON-NLS - //critical core/system error and process needs to be interrupted - hasCritError = true; - errorList.add(ex.getMessage()); - } catch (TskDataException ex) { - logger.log(Level.WARNING, "Data errors occurred while running add image. ", ex); //NON-NLS - errorList.add(ex.getMessage()); - } - - // handle addImage done - postProcess(); - - // unclock the DB - SleuthkitCase.releaseExclusiveLock(); + currentCase.getSleuthkitCase().acquireExclusiveLock(); + addImageProcess = currentCase.makeAddImageProcess(timeZone, true, noFatOrphans); + dirFetcher = new Thread( new CurrentDirectoryFetcher(progressMonitor, addImageProcess)); + try { + progressMonitor.setIndeterminate(true); + progressMonitor.setProgress(0); + dirFetcher.start(); + addImageProcess.run(new String[]{this.imagePath}); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Core errors occurred while running add image. ", ex); //NON-NLS + hasCritError = true; + errorList.add(ex.getMessage()); + } catch (TskDataException ex) { + logger.log(Level.WARNING, "Data errors occurred while running add image. ", ex); //NON-NLS + errorList.add(ex.getMessage()); + } + postProcess(); + } finally { + currentCase.getSleuthkitCase().releaseExclusiveLock(); + } } /** From c00db4b4f6427e6df1409959431f782109f1acb0 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 5 Aug 2014 16:19:24 -0400 Subject: [PATCH 10/16] Fix bug in DeletedContent.DeletedContentChildren.createKeys() --- Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java index 80d28d96a0..ca9577b3cf 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java @@ -226,6 +226,7 @@ public class DeletedContent implements AutopsyVisitableItem { protected boolean createKeys(List list) { List queryList = runFsQuery(); if (queryList.size() == MAX_OBJECTS) { + queryList.remove(queryList.size() - 1); SwingUtilities.invokeLater(new Runnable() { @Override public void run() { @@ -236,7 +237,6 @@ public class DeletedContent implements AutopsyVisitableItem { }); } - queryList.remove(queryList.size() - 1); list.addAll(queryList); return true; } From 1eb456baeb1b8a21c889b44fa7b3767708af8b17 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 5 Aug 2014 16:20:06 -0400 Subject: [PATCH 11/16] Fix bug in DeletedContent.DeletedContentChildren.createKeys() --- Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java index 80d28d96a0..5e5ac5ffd5 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java @@ -226,6 +226,7 @@ public class DeletedContent implements AutopsyVisitableItem { protected boolean createKeys(List list) { List queryList = runFsQuery(); if (queryList.size() == MAX_OBJECTS) { + queryList.remove(queryList.size() - 1); SwingUtilities.invokeLater(new Runnable() { @Override public void run() { @@ -235,8 +236,6 @@ public class DeletedContent implements AutopsyVisitableItem { } }); } - - queryList.remove(queryList.size() - 1); list.addAll(queryList); return true; } From faf592a2bb3cb08e0e72c4824e2171e92b0156e1 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 6 Aug 2014 17:59:19 -0400 Subject: [PATCH 12/16] Add file times and size properties to ContentTagNode property sheet --- .../autopsy/datamodel/Bundle.properties | 10 +++++ .../autopsy/datamodel/ContentTagNode.java | 45 ++++++++++++++----- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties index 378c308335..0966a935b2 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties @@ -58,6 +58,16 @@ ContentTagNode.createSheet.filePath.name=File Path ContentTagNode.createSheet.filePath.displayName=File Path ContentTagNode.createSheet.comment.name=Comment ContentTagNode.createSheet.comment.displayName=Comment +ContentTagNode.createSheet.fileModifiedTime.name=Modified Time +ContentTagNode.createSheet.fileModifiedTime.displayName=Modified Time +ContentTagNode.createSheet.fileChangedTime.name=Changed Time +ContentTagNode.createSheet.fileChangedTime.displayName=Changed Time +ContentTagNode.createSheet.fileAccessedTime.name=Accessed Time +ContentTagNode.createSheet.fileAccessedTime.displayName=Accessed Time +ContentTagNode.createSheet.fileCreatedTime.name=Created Time +ContentTagNode.createSheet.fileCreatedTime.displayName=Created Time +ContentTagNode.createSheet.filesize.name=Size +ContentTagNode.createSheet.filesize.displayName=Size ContentTagTypeNode.displayName.text=File Tags ContentTagTypeNode.createSheet.name.name=Name ContentTagTypeNode.createSheet.name.displayName=Name diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java index 31343f6d81..30bfb573f4 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java @@ -20,13 +20,15 @@ package org.sleuthkit.autopsy.datamodel; import java.util.List; import java.util.logging.Level; -import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.Action; import org.openide.nodes.Children; import org.openide.nodes.Sheet; import org.openide.util.NbBundle; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.actions.DeleteContentTagAction; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.TskCoreException; @@ -51,24 +53,26 @@ class ContentTagNode extends DisplayableItemNode { @Override protected Sheet createSheet() { + Content content = tag.getContent(); + String contentPath; + try { + contentPath = content.getUniquePath(); + } catch (TskCoreException ex) { + Logger.getLogger(ContentTagNode.class.getName()).log(Level.SEVERE, "Failed to get path for content (id = " + content.getId() + ")", ex); //NON-NLS + contentPath = NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.unavail.path"); + } + AbstractFile file = content instanceof AbstractFile ? (AbstractFile)content : null; + Sheet propertySheet = super.createSheet(); Sheet.Set properties = propertySheet.get(Sheet.PROPERTIES); if (properties == null) { properties = Sheet.createPropertiesSet(); propertySheet.put(properties); } - properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.file.name"), NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.file.displayName"), "", - tag.getContent().getName())); - String contentPath; - try { - contentPath = tag.getContent().getUniquePath(); - } catch (TskCoreException ex) { - Logger.getLogger(ContentTagNode.class.getName()).log(Level.SEVERE, "Failed to get path for content (id = " + tag.getContent().getId() + ")", ex); //NON-NLS - contentPath = NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.unavail.path"); - } + content.getName())); properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.filePath.name"), NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.filePath.displayName"), "", @@ -77,7 +81,26 @@ class ContentTagNode extends DisplayableItemNode { NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.comment.displayName"), "", tag.getComment())); - + properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileModifiedTime.name"), + NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileModifiedTime.displayName"), + "", + file != null ? ContentUtils.getStringTime(file.getMtime(), file) : "")); + properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileChangedTime.name"), + NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileChangedTime.displayName"), + "", + file != null ? ContentUtils.getStringTime(file.getCtime(), file) : "")); + properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileAccessedTime.name"), + NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileAccessedTime.displayName"), + "", + file != null ? ContentUtils.getStringTime(file.getAtime(), file) : "")); + properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileCreatedTime.name"), + NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileCreatedTime.displayName"), + "", + file != null ? ContentUtils.getStringTime(file.getCrtime(), file) : "")); + properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.filesize.name"), + NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.filesize.displayName"), + "", + content.getSize())); return propertySheet; } From 01749ab3abc38c364c5a9fcbe528ef86f428c7db Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 6 Aug 2014 18:01:30 -0400 Subject: [PATCH 13/16] Add file times and size properties to ContentTagNode property sheet --- Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties | 4 ++-- Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties index 0966a935b2..a9cbac245a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties @@ -66,8 +66,8 @@ ContentTagNode.createSheet.fileAccessedTime.name=Accessed Time ContentTagNode.createSheet.fileAccessedTime.displayName=Accessed Time ContentTagNode.createSheet.fileCreatedTime.name=Created Time ContentTagNode.createSheet.fileCreatedTime.displayName=Created Time -ContentTagNode.createSheet.filesize.name=Size -ContentTagNode.createSheet.filesize.displayName=Size +ContentTagNode.createSheet.fileSize.name=Size +ContentTagNode.createSheet.fileSize.displayName=Size ContentTagTypeNode.displayName.text=File Tags ContentTagTypeNode.createSheet.name.name=Name ContentTagTypeNode.createSheet.name.displayName=Name diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java index 30bfb573f4..62d56c09f9 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentTagNode.java @@ -97,8 +97,8 @@ class ContentTagNode extends DisplayableItemNode { NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileCreatedTime.displayName"), "", file != null ? ContentUtils.getStringTime(file.getCrtime(), file) : "")); - properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.filesize.name"), - NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.filesize.displayName"), + properties.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileSize.name"), + NbBundle.getMessage(this.getClass(), "ContentTagNode.createSheet.fileSize.displayName"), "", content.getSize())); return propertySheet; From 7466fb0a6d3266de2d9a8059fc0de819cbd35bce Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 8 Aug 2014 10:46:57 -0400 Subject: [PATCH 14/16] Fixed timing issue in scheduler where file could be stuck in running state even if it had been completed --- .../sleuthkit/autopsy/ingest/IngestJob.java | 76 ++++++++++--------- .../autopsy/ingest/IngestScheduler.java | 55 +++++++------- 2 files changed, 69 insertions(+), 62 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 0cdcdbeb24..f885b5c171 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -193,46 +193,54 @@ final class IngestJob { } void process(DataSourceIngestTask task) throws InterruptedException { - if (!isCancelled() && !dataSourceIngestPipeline.isEmpty()) { - List errors = new ArrayList<>(); - errors.addAll(dataSourceIngestPipeline.process(task, dataSourceIngestProgress)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - } - if (null != dataSourceIngestProgress) { - dataSourceIngestProgress.finish(); - // This is safe because this method will be called at most once per - // ingest job and finish() will not be called while that single - // data source ingest task has not been reported complete by this - // code to the ingest scheduler. - dataSourceIngestProgress = null; - } - ingestTaskScheduler.notifyTaskCompleted(task); - } - - void process(FileIngestTask task) throws InterruptedException { - if (!isCancelled()) { - FileIngestPipeline pipeline = fileIngestPipelines.take(); - if (!pipeline.isEmpty()) { - AbstractFile file = task.getFile(); - synchronized (this) { - ++processedFiles; - if (processedFiles <= estimatedFilesToProcess) { - fileIngestProgress.progress(file.getName(), (int) processedFiles); - } else { - fileIngestProgress.progress(file.getName(), (int) estimatedFilesToProcess); - } - } + try { + if (!isCancelled() && !dataSourceIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); + errors.addAll(dataSourceIngestPipeline.process(task, dataSourceIngestProgress)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } } - fileIngestPipelines.put(pipeline); + if (null != dataSourceIngestProgress) { + dataSourceIngestProgress.finish(); + // This is safe because this method will be called at most once per + // ingest job and finish() will not be called while that single + // data source ingest task has not been reported complete by this + // code to the ingest scheduler. + dataSourceIngestProgress = null; + } + } + finally { + ingestTaskScheduler.notifyTaskCompleted(task); + } + } + + void process(FileIngestTask task) throws InterruptedException { + try { + if (!isCancelled()) { + FileIngestPipeline pipeline = fileIngestPipelines.take(); + if (!pipeline.isEmpty()) { + AbstractFile file = task.getFile(); + synchronized (this) { + ++processedFiles; + if (processedFiles <= estimatedFilesToProcess) { + fileIngestProgress.progress(file.getName(), (int) processedFiles); + } else { + fileIngestProgress.progress(file.getName(), (int) estimatedFilesToProcess); + } + } + List errors = new ArrayList<>(); + errors.addAll(pipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + fileIngestPipelines.put(pipeline); + } + } + finally { + ingestTaskScheduler.notifyTaskCompleted(task); } - ingestTaskScheduler.notifyTaskCompleted(task); } void finish() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 5f74bb68c8..b5338cd773 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -52,30 +52,34 @@ final class IngestScheduler { private final AtomicLong nextIngestJobId = new AtomicLong(0L); private final ConcurrentHashMap ingestJobsById = new ConcurrentHashMap<>(); private volatile boolean enabled = false; -// private volatile boolean cancellingAllTasks = false; TODO: Uncomment this with related code, if desired + // private volatile boolean cancellingAllTasks = false; TODO: Uncomment this with related code, if desired private final DataSourceIngestTaskQueue dataSourceTaskDispenser = new DataSourceIngestTaskQueue(); private final FileIngestTaskQueue fileTaskDispenser = new FileIngestTaskQueue(); + // The following five collections lie at the heart of the scheduler. - // // The pending tasks queues are used to schedule tasks for an ingest job. If // multiple jobs are scheduled, tasks from different jobs may become - // interleaved in these queues. Data source tasks go into a simple FIFO - // queue that is consumed by the ingest threads. File tasks are "shuffled" + // interleaved in these queues. + + // FIFO queue for data source-level tasks. + private final LinkedBlockingQueue pendingDataSourceTasks = new LinkedBlockingQueue<>(); // Guarded by this + + // File tasks are "shuffled" // through root directory (priority queue), directory (LIFO), and file tasks // queues (LIFO). If a file task makes it into the pending file tasks queue, // it is consumed by the ingest threads. - // - // The "tasks in progress" list is used to determine when an ingest job is - // completed and should be shut down, i.e., the job should shut down its - // ingest pipelines and finish its progress bars. Tasks stay in the "tasks - // in progress" list either until discarded by the scheduler or the ingest - // thread that is working on the task notifies the scheduler that the task - // is completed. - private final LinkedBlockingQueue pendingDataSourceTasks = new LinkedBlockingQueue<>(); private final TreeSet pendingRootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); // Guarded by this private final List pendingDirectoryTasks = new ArrayList<>(); // Guarded by this - private final BlockingDeque pendingFileTasks = new LinkedBlockingDeque<>(); - private final List tasksInProgress = new ArrayList<>(); // Guarded by this + private final BlockingDeque pendingFileTasks = new LinkedBlockingDeque<>(); // Not guarded + + // The "tasks in progress" list has: + // - File and data source tasks that are running + // - File tasks that are in the pending file queue + // It is used to determine when a job is done. It has both pending and running + // tasks because we do not lock the 'pendingFileTasks' and a task needs to be in + // at least one of the pending or inprogress lists at all times before it is completed. + // files are added to this when the are added to pendingFilesTasks and removed when they complete + private final List tasksInProgressAndPending = new ArrayList<>(); // Guarded by this synchronized static IngestScheduler getInstance() { if (instance == null) { @@ -133,12 +137,12 @@ final class IngestScheduler { synchronized private void scheduleDataSourceIngestTask(IngestJob job) throws InterruptedException { DataSourceIngestTask task = new DataSourceIngestTask(job); - tasksInProgress.add(task); + tasksInProgressAndPending.add(task); try { // Should not block, queue is (theoretically) unbounded. pendingDataSourceTasks.put(task); } catch (InterruptedException ex) { - tasksInProgress.remove(task); + tasksInProgressAndPending.remove(task); Logger.getLogger(IngestScheduler.class.getName()).log(Level.SEVERE, "Interruption of unexpected block on pending data source tasks queue", ex); //NON-NLS throw ex; } @@ -149,7 +153,6 @@ final class IngestScheduler { for (AbstractFile firstLevelFile : topLevelFiles) { FileIngestTask task = new FileIngestTask(job, firstLevelFile); if (shouldEnqueueFileTask(task)) { - tasksInProgress.add(task); pendingRootDirectoryTasks.add(task); } } @@ -212,9 +215,7 @@ final class IngestScheduler { if (shouldEnqueueFileTask(directoryTask)) { addToPendingFileTasksQueue(directoryTask); tasksEnqueuedForDirectory = true; - } else { - tasksInProgress.remove(directoryTask); - } + } // If the directory contains subdirectories or files, try to // enqueue tasks for them as well. @@ -227,13 +228,11 @@ final class IngestScheduler { if (file.hasChildren()) { // Found a subdirectory, put the task in the // pending directory tasks queue. - tasksInProgress.add(childTask); pendingDirectoryTasks.add(childTask); tasksEnqueuedForDirectory = true; } else if (shouldEnqueueFileTask(childTask)) { // Found a file, put the task directly into the // pending file tasks queue. - tasksInProgress.add(childTask); addToPendingFileTasksQueue(childTask); tasksEnqueuedForDirectory = true; } @@ -304,6 +303,7 @@ final class IngestScheduler { } synchronized private void addToPendingFileTasksQueue(FileIngestTask task) throws IllegalStateException { + tasksInProgressAndPending.add(task); try { // Should not block, queue is (theoretically) unbounded. /* add to top of list because we had one image that had a folder with @@ -313,7 +313,7 @@ final class IngestScheduler { */ pendingFileTasks.addFirst(task); } catch (IllegalStateException ex) { - tasksInProgress.remove(task); + tasksInProgressAndPending.remove(task); Logger.getLogger(IngestScheduler.class.getName()).log(Level.SEVERE, "Interruption of unexpected block on pending file tasks queue", ex); //NON-NLS throw ex; } @@ -326,7 +326,6 @@ final class IngestScheduler { // Send the file task directly to file tasks queue, no need to // update the pending root directory or pending directory tasks // queues. - tasksInProgress.add(task); addToPendingFileTasksQueue(task); } } @@ -344,7 +343,7 @@ final class IngestScheduler { boolean jobIsCompleted; IngestJob job = task.getIngestJob(); synchronized (this) { - tasksInProgress.remove(task); + tasksInProgressAndPending.remove(task); jobIsCompleted = ingestJobIsComplete(job); } if (jobIsCompleted) { @@ -382,7 +381,7 @@ final class IngestScheduler { while (iterator.hasNext()) { IngestTask task = (IngestTask) iterator.next(); if (task.getIngestJob().getId() == jobId) { - tasksInProgress.remove((IngestTask) task); + tasksInProgressAndPending.remove((IngestTask) task); iterator.remove(); } } @@ -420,13 +419,13 @@ final class IngestScheduler { synchronized private void removeAllPendingTasks(Collection taskQueue) { Iterator iterator = taskQueue.iterator(); while (iterator.hasNext()) { - tasksInProgress.remove((IngestTask) iterator.next()); + tasksInProgressAndPending.remove((IngestTask) iterator.next()); iterator.remove(); } } synchronized private boolean ingestJobIsComplete(IngestJob job) { - for (IngestTask task : tasksInProgress) { + for (IngestTask task : tasksInProgressAndPending) { if (task.getIngestJob().getId() == job.getId()) { return false; } From 11da944b651faa93c4d5652054bffc5c1a277ea7 Mon Sep 17 00:00:00 2001 From: jmillman Date: Fri, 8 Aug 2014 15:28:22 -0400 Subject: [PATCH 15/16] use strftime to get datetime as seconds since unix epoch --- .../autopsy/modules/android/WWFMessageAnalyzer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java index 1abe891f09..b67c86fa8d 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java @@ -80,7 +80,7 @@ class WWFMessageAnalyzer { try { resultSet = statement.executeQuery( - "SELECT message,created_at,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); + "SELECT message,strftime('%s' ,created_at) as datetime,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); String message; // WWF Message String user_id; // the ID of the user who sent the message. @@ -88,7 +88,8 @@ class WWFMessageAnalyzer { while (resultSet.next()) { message = resultSet.getString("message"); - Long created_at = Long.valueOf(resultSet.getString("created_at")) / 1000; +// new SimpleDateFormat("yyyy-MM-ddTHH:mm:ss z"). + Long created_at = resultSet.getLong("datetime"); user_id = resultSet.getString("user_id"); game_id = resultSet.getString("game_id"); From 70c763d3005477eceed97722e7de120d7e49c4f1 Mon Sep 17 00:00:00 2001 From: jmillman Date: Mon, 11 Aug 2014 11:13:00 -0400 Subject: [PATCH 16/16] removed commented dead code --- .../sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java index b67c86fa8d..3bda358a24 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java @@ -88,7 +88,6 @@ class WWFMessageAnalyzer { while (resultSet.next()) { message = resultSet.getString("message"); -// new SimpleDateFormat("yyyy-MM-ddTHH:mm:ss z"). Long created_at = resultSet.getLong("datetime"); user_id = resultSet.getString("user_id"); game_id = resultSet.getString("game_id");