diff --git a/Core/build.xml b/Core/build.xml
index cd3b394f24..a8fa08cb0d 100644
--- a/Core/build.xml
+++ b/Core/build.xml
@@ -6,7 +6,8 @@
Builds, tests, and runs the project org.sleuthkit.autopsy.core
-
+
+
@@ -18,9 +19,7 @@
-
-
-
+
@@ -58,8 +57,8 @@
-
+
recentlyAddedCeArtifacts = new LinkedHashSet<>();
- private static int ceModuleInstanceCount = 0;
+ private static int correlationModuleInstanceCount;
+ private static boolean flagNotableItems;
private final ExecutorService jobProcessingExecutor;
private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d";
private final PropertyChangeListener pcl1 = new IngestModuleEventListener();
@@ -88,21 +89,20 @@ public class IngestEventsListener {
}
/**
- * Enable this IngestEventsListener to add contents to the Correlation
- * Engine.
- *
+ * Increase the number of IngestEventsListeners adding contents to the
+ * Correlation Engine.
*/
public synchronized static void incrementCorrelationEngineModuleCount() {
- ceModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method.
+ correlationModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method.
}
/**
- * Disable this IngestEventsListener from adding contents to the Correlation
- * Engine.
+ * Decrease the number of IngestEventsListeners adding contents to the
+ * Correlation Engine.
*/
public synchronized static void decrementCorrelationEngineModuleCount() {
if (getCeModuleInstanceCount() > 0) { //prevent it ingestJobCounter from going negative
- ceModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method.
+ correlationModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method.
}
}
@@ -111,17 +111,35 @@ public class IngestEventsListener {
* is being run during injest to 0.
*/
synchronized static void resetCeModuleInstanceCount() {
- ceModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset
+ correlationModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset
}
/**
- * Wether or not the Correlation Engine Module is enabled for any of the
+ * Whether or not the Correlation Engine Module is enabled for any of the
* currently running ingest jobs.
*
* @return boolean True for Correlation Engine enabled, False for disabled
*/
- private synchronized static int getCeModuleInstanceCount() {
- return ceModuleInstanceCount;
+ public synchronized static int getCeModuleInstanceCount() {
+ return correlationModuleInstanceCount;
+ }
+
+ /**
+ * Are notable items being flagged?
+ *
+ * @return True if flagging notable items; otherwise false.
+ */
+ public synchronized static boolean isFlagNotableItems() {
+ return flagNotableItems;
+ }
+
+ /**
+ * Configure the listener to flag notable items or not.
+ *
+ * @param value True to flag notable items; otherwise false.
+ */
+ public synchronized static void setFlagNotableItems(boolean value) {
+ flagNotableItems = value;
}
@NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
@@ -174,7 +192,7 @@ public class IngestEventsListener {
}
switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) {
case DATA_ADDED: {
- jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt));
+ jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, isFlagNotableItems()));
break;
}
}
@@ -212,10 +230,12 @@ public class IngestEventsListener {
private final EamDb dbManager;
private final PropertyChangeEvent event;
+ private final boolean flagNotableItemsEnabled;
- private DataAddedTask(EamDb db, PropertyChangeEvent evt) {
+ private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled) {
dbManager = db;
event = evt;
+ this.flagNotableItemsEnabled = flagNotableItemsEnabled;
}
@Override
@@ -241,10 +261,12 @@ public class IngestEventsListener {
// query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad".
// if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case,
// create TSK_INTERESTING_ARTIFACT_HIT artifact on BB.
- List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
- if (!caseDisplayNames.isEmpty()) {
- postCorrelatedBadArtifactToBlackboard(bbArtifact,
- caseDisplayNames);
+ if (flagNotableItemsEnabled) {
+ List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
+ if (!caseDisplayNames.isEmpty()) {
+ postCorrelatedBadArtifactToBlackboard(bbArtifact,
+ caseDisplayNames);
+ }
}
eamArtifacts.add(eamArtifact);
}
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties
new file mode 100755
index 0000000000..a525713f7c
--- /dev/null
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties
@@ -0,0 +1,2 @@
+IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings
+IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java
index 9657cdcd19..991da1ad58 100644
--- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java
@@ -56,9 +56,11 @@ import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListen
*/
@Messages({"IngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
"IngestModule.prevCaseComment.text=Previous Case: "})
-class IngestModule implements FileIngestModule {
+final class IngestModule implements FileIngestModule {
- private final static Logger LOGGER = Logger.getLogger(IngestModule.class.getName());
+ static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = true;
+
+ private final static Logger logger = Logger.getLogger(IngestModule.class.getName());
private final IngestServices services = IngestServices.getInstance();
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
@@ -68,8 +70,19 @@ class IngestModule implements FileIngestModule {
private Blackboard blackboard;
private CorrelationAttribute.Type filesType;
+ private final boolean flagTaggedNotableItems;
+
+ /**
+ * Instantiate the Correlation Engine ingest module.
+ *
+ * @param settings The ingest settings for the module instance.
+ */
+ IngestModule(IngestSettings settings) {
+ flagTaggedNotableItems = settings.isFlagTaggedNotableItems();
+ }
+
@Override
- public ProcessResult process(AbstractFile af) {
+ public ProcessResult process(AbstractFile abstractFile) {
if (EamDb.isEnabled() == false) {
/*
* Not signaling an error for now. This is a workaround for the way
@@ -83,11 +96,11 @@ class IngestModule implements FileIngestModule {
try {
blackboard = Case.getOpenCase().getServices().getBlackboard();
} catch (NoCurrentCaseException ex) {
- LOGGER.log(Level.SEVERE, "Exception while getting open case.", ex);
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex);
return ProcessResult.ERROR;
}
- if (!EamArtifactUtil.isValidCentralRepoFile(af)) {
+ if (!EamArtifactUtil.isValidCentralRepoFile(abstractFile)) {
return ProcessResult.OK;
}
@@ -95,7 +108,7 @@ class IngestModule implements FileIngestModule {
try {
dbManager = EamDb.getInstance();
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
+ logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
return ProcessResult.ERROR;
}
@@ -105,21 +118,23 @@ class IngestModule implements FileIngestModule {
}
// get the hash because we're going to correlate it
- String md5 = af.getMd5Hash();
+ String md5 = abstractFile.getMd5Hash();
if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) {
return ProcessResult.OK;
}
- /* Search the central repo to see if this file was previously
- * marked as being bad. Create artifact if it was. */
- if (af.getKnown() != TskData.FileKnown.KNOWN) {
+ /*
+ * Search the central repo to see if this file was previously marked as
+ * being bad. Create artifact if it was.
+ */
+ if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
try {
- List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
- if (!caseDisplayNames.isEmpty()) {
- postCorrelatedBadFileToBlackboard(af, caseDisplayNames);
+ List caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
+ if (!caseDisplayNamesList.isEmpty()) {
+ postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList);
}
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS
return ProcessResult.ERROR;
}
}
@@ -130,14 +145,14 @@ class IngestModule implements FileIngestModule {
CorrelationAttributeInstance cefi = new CorrelationAttributeInstance(
eamCase,
eamDataSource,
- af.getParentPath() + af.getName(),
+ abstractFile.getParentPath() + abstractFile.getName(),
null,
- TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
+ TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
);
eamArtifact.addInstance(cefi);
dbManager.prepareBulkArtifact(eamArtifact);
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
return ProcessResult.ERROR;
}
@@ -147,6 +162,7 @@ class IngestModule implements FileIngestModule {
@Override
public void shutDown() {
IngestEventsListener.decrementCorrelationEngineModuleCount();
+
if ((EamDb.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) {
return;
}
@@ -154,19 +170,19 @@ class IngestModule implements FileIngestModule {
try {
dbManager = EamDb.getInstance();
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
+ logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
return;
}
try {
dbManager.bulkInsertArtifacts();
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS
}
try {
Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamCase.getCaseUUID(), eamDataSource.getDeviceID());
- LOGGER.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS
+ logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS
}
// TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk.
@@ -181,6 +197,25 @@ class IngestModule implements FileIngestModule {
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
IngestEventsListener.incrementCorrelationEngineModuleCount();
+
+ /*
+ * Tell the IngestEventsListener to flag notable items based on the
+ * current module's configuration. This is a work around for the lack of
+ * an artifacts pipeline. Note that this can be changed by another
+ * module instance. All modules are affected by the value. While not
+ * ideal, this will be good enough until a better solution can be
+ * posited.
+ *
+ * Note: Flagging cannot be disabled if any other instances of the
+ * Correlation Engine module are running. This restriction is to prevent
+ * missing results in the case where the first module is flagging
+ * notable items, and the proceeding module (with flagging disabled)
+ * causes the first to stop flagging.
+ */
+ if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagNotableItems()) {
+ IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems);
+ }
+
if (EamDb.isEnabled() == false) {
/*
* Not throwing the customary exception for now. This is a
@@ -200,14 +235,14 @@ class IngestModule implements FileIngestModule {
try {
autopsyCase = Case.getOpenCase();
} catch (NoCurrentCaseException ex) {
- LOGGER.log(Level.SEVERE, "Exception while getting open case.", ex);
- throw new IngestModuleException("Exception while getting open case.", ex);
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex);
+ throw new IngestModuleException("Exception while getting open case.", ex);
}
-
+
// Don't allow sqlite central repo databases to be used for multi user cases
if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE)
&& (EamDbPlatformEnum.getSelectedPlatform() == EamDbPlatformEnum.SQLITE)) {
- LOGGER.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository.");
+ logger.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository.");
throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS
}
jobId = context.getJobId();
@@ -216,14 +251,14 @@ class IngestModule implements FileIngestModule {
try {
centralRepoDb = EamDb.getInstance();
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS
throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS
}
try {
filesType = centralRepoDb.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID);
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
}
@@ -237,15 +272,15 @@ class IngestModule implements FileIngestModule {
try {
eamCase = centralRepoDb.newCase(autopsyCase);
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error creating new case in ingest module start up.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error creating new case in ingest module start up.", ex); // NON-NLS
throw new IngestModuleException("Error creating new case in ingest module start up.", ex); // NON-NLS
}
}
-
+
try {
eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource());
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS
throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS
}
// TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter
@@ -259,7 +294,7 @@ class IngestModule implements FileIngestModule {
centralRepoDb.newDataSource(eamDataSource);
}
} catch (EamDbException ex) {
- LOGGER.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS
throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS
}
@@ -282,7 +317,7 @@ class IngestModule implements FileIngestModule {
// index the artifact for keyword search
blackboard.indexArtifact(tifArtifact);
} catch (Blackboard.BlackboardException ex) {
- LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
+ logger.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
}
// send inbox message
@@ -291,9 +326,9 @@ class IngestModule implements FileIngestModule {
// fire event to notify UI of this new artifact
services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT));
} catch (TskCoreException ex) {
- LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
} catch (IllegalStateException ex) {
- LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
+ logger.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
}
}
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java
index ed3d4f0915..6ef03ae00d 100644
--- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java
@@ -1,7 +1,7 @@
/*
* Central Repository
*
- * Copyright 2015-2017 Basis Technology Corp.
+ * Copyright 2015-2018 Basis Technology Corp.
* Contact: carrier sleuthkit org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,6 +25,9 @@ import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter;
import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel;
+import org.sleuthkit.autopsy.coreutils.Version;
+import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
+import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings;
/**
* Factory for Central Repository ingest modules
@@ -34,8 +37,11 @@ import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel;
"IngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"})
public class IngestModuleFactory extends IngestModuleFactoryAdapter {
- private static final String VERSION_NUMBER = "0.8.0";
-
+ /**
+ * Get the name of the module.
+ *
+ * @return The module name.
+ */
static String getModuleName() {
return Bundle.IngestModuleFactory_ingestmodule_name();
}
@@ -52,7 +58,7 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter {
@Override
public String getModuleVersionNumber() {
- return VERSION_NUMBER;
+ return Version.getVersion();
}
@Override
@@ -61,8 +67,8 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter {
}
@Override
- public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) {
- return new IngestModule();
+ public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) {
+ return new IngestModule((IngestSettings) settings);
}
@Override
@@ -76,5 +82,30 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter {
globalOptionsPanel.load();
return globalOptionsPanel;
}
+
+ @Override
+ public IngestModuleIngestJobSettings getDefaultIngestJobSettings() {
+ return new IngestSettings();
+ }
+
+ @Override
+ public boolean hasIngestJobSettingsPanel() {
+ return true;
+ }
+
+ @Override
+ public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) {
+ if (settings instanceof IngestSettings) {
+ return new IngestSettingsPanel((IngestSettings) settings);
+ }
+ /*
+ * Compatibility check for older versions.
+ */
+ if (settings instanceof NoIngestModuleIngestJobSettings) {
+ return new IngestSettingsPanel(new IngestSettings());
+ }
+
+ throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings");
+ }
}
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java
new file mode 100755
index 0000000000..32ab9e9f2d
--- /dev/null
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java
@@ -0,0 +1,71 @@
+/*
+ * Central Repository
+ *
+ * Copyright 2018 Basis Technology Corp.
+ * Contact: carrier sleuthkit org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.centralrepository.ingestmodule;
+
+import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
+
+/**
+ * Ingest job settings for the Correlation Engine module.
+ */
+final class IngestSettings implements IngestModuleIngestJobSettings {
+
+ private static final long serialVersionUID = 1L;
+
+ private boolean flagTaggedNotableItems;
+
+ /**
+ * Instantiate the ingest job settings with default values.
+ */
+ IngestSettings() {
+ this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS;
+ }
+
+ /**
+ * Instantiate the ingest job settings.
+ *
+ * @param flagTaggedNotableItems Flag previously tagged notable items.
+ */
+ IngestSettings(boolean flagTaggedNotableItems) {
+ this.flagTaggedNotableItems = flagTaggedNotableItems;
+ }
+
+ @Override
+ public long getVersionNumber() {
+ return serialVersionUID;
+ }
+
+ /**
+ * Are previously tagged notable items to be flagged?
+ *
+ * @return True if flagging; otherwise false.
+ */
+ boolean isFlagTaggedNotableItems() {
+ return flagTaggedNotableItems;
+ }
+
+ /**
+ * Flag or ignore previously identified notable items.
+ *
+ * @param ignorePreviousNotableItems Are previously tagged notable items to
+ * be flagged?
+ */
+ void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) {
+ this.flagTaggedNotableItems = flagTaggedNotableItems;
+ }
+}
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form
new file mode 100755
index 0000000000..564031cb72
--- /dev/null
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form
@@ -0,0 +1,63 @@
+
+
+
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java
new file mode 100755
index 0000000000..57d4f0a098
--- /dev/null
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java
@@ -0,0 +1,97 @@
+/*
+ * Central Repository
+ *
+ * Copyright 2018 Basis Technology Corp.
+ * Contact: carrier sleuthkit org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.centralrepository.ingestmodule;
+
+import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
+import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
+
+/**
+ * Ingest job settings panel for the Correlation Engine module.
+ */
+final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel {
+
+ /**
+ * Creates new form IngestSettingsPanel
+ */
+ public IngestSettingsPanel(IngestSettings settings) {
+ initComponents();
+ customizeComponents(settings);
+ }
+
+ /**
+ * Update components with values from the ingest job settings.
+ *
+ * @param settings The ingest job settings.
+ */
+ private void customizeComponents(IngestSettings settings) {
+ flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems());
+ }
+
+ @Override
+ public IngestModuleIngestJobSettings getSettings() {
+ return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected());
+ }
+
+ /**
+ * This method is called from within the constructor to initialize the form.
+ * WARNING: Do NOT modify this code. The content of this method is always
+ * regenerated by the Form Editor.
+ */
+ @SuppressWarnings("unchecked")
+ // //GEN-BEGIN:initComponents
+ private void initComponents() {
+
+ ingestSettingsLabel = new javax.swing.JLabel();
+ flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox();
+
+ ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N
+ org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N
+
+ org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N
+
+ javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
+ this.setLayout(layout);
+ layout.setHorizontalGroup(
+ layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+ .addGroup(layout.createSequentialGroup()
+ .addContainerGap()
+ .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+ .addGroup(layout.createSequentialGroup()
+ .addGap(10, 10, 10)
+ .addComponent(flagTaggedNotableItemsCheckbox))
+ .addComponent(ingestSettingsLabel))
+ .addContainerGap(65, Short.MAX_VALUE))
+ );
+ layout.setVerticalGroup(
+ layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+ .addGroup(layout.createSequentialGroup()
+ .addContainerGap()
+ .addComponent(ingestSettingsLabel)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
+ .addComponent(flagTaggedNotableItemsCheckbox)
+ .addContainerGap(245, Short.MAX_VALUE))
+ );
+ }// //GEN-END:initComponents
+
+ // Variables declaration - do not modify//GEN-BEGIN:variables
+ private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox;
+ private javax.swing.JLabel ingestSettingsLabel;
+ // End of variables declaration//GEN-END:variables
+
+}
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form
index ea06641799..50979ba938 100644
--- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form
@@ -44,7 +44,7 @@
-
+
@@ -133,7 +133,7 @@
-
+
@@ -173,29 +173,29 @@
-
+
-
+
-
+
-
+
-
+
diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java
index aaf6c6b4f7..55ea941d33 100644
--- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java
+++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java
@@ -243,7 +243,7 @@ public class EamDbSettingsDialog extends JDialog {
.addGroup(pnSQLiteSettingsLayout.createSequentialGroup()
.addComponent(cbDatabaseType, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
- .addComponent(lbSingleUserSqLite, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
+ .addComponent(lbSingleUserSqLite, javax.swing.GroupLayout.DEFAULT_SIZE, 467, Short.MAX_VALUE)
.addGap(9, 9, 9))
.addGroup(pnSQLiteSettingsLayout.createSequentialGroup()
.addComponent(tfDatabasePath)
@@ -273,25 +273,25 @@ public class EamDbSettingsDialog extends JDialog {
.addComponent(lbDatabaseType, javax.swing.GroupLayout.Alignment.TRAILING))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
- .addComponent(lbDatabasePath, javax.swing.GroupLayout.PREFERRED_SIZE, 23, javax.swing.GroupLayout.PREFERRED_SIZE)
+ .addComponent(lbDatabasePath)
.addComponent(tfDatabasePath, javax.swing.GroupLayout.PREFERRED_SIZE, 23, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(bnDatabasePathFileOpen))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(tbDbHostname, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
- .addComponent(lbHostName, javax.swing.GroupLayout.PREFERRED_SIZE, 22, javax.swing.GroupLayout.PREFERRED_SIZE))
+ .addComponent(lbHostName))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(tbDbPort, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
- .addComponent(lbPort, javax.swing.GroupLayout.PREFERRED_SIZE, 20, javax.swing.GroupLayout.PREFERRED_SIZE))
+ .addComponent(lbPort))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(tbDbUsername, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
- .addComponent(lbUserName, javax.swing.GroupLayout.PREFERRED_SIZE, 20, javax.swing.GroupLayout.PREFERRED_SIZE))
+ .addComponent(lbUserName))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jpDbPassword, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
- .addComponent(lbUserPassword, javax.swing.GroupLayout.PREFERRED_SIZE, 20, javax.swing.GroupLayout.PREFERRED_SIZE))
+ .addComponent(lbUserPassword))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(lbFullDbPath, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
@@ -317,7 +317,7 @@ public class EamDbSettingsDialog extends JDialog {
.addGroup(layout.createSequentialGroup()
.addGap(10, 10, 10)
.addComponent(pnSQLiteSettings, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
- .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 11, Short.MAX_VALUE)
.addComponent(pnButtons, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addGap(10, 10, 10))
);
diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java
index 583c2aa157..99692ea633 100644
--- a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java
+++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java
@@ -23,12 +23,14 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.logging.Level;
import javax.swing.AbstractAction;
import javax.swing.Action;
import org.openide.util.NbBundle;
import org.openide.util.Utilities;
import org.sleuthkit.autopsy.actions.AddContentTagAction;
import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction;
+import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
@@ -41,6 +43,7 @@ import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.LocalDirectory;
import org.sleuthkit.datamodel.VirtualDirectory;
import org.sleuthkit.datamodel.Volume;
+import org.sleuthkit.autopsy.coreutils.Logger;
public class ExplorerNodeActionVisitor extends ContentVisitor.Default> {
@@ -71,8 +74,12 @@ public class ExplorerNodeActionVisitor extends ContentVisitor.Default visit(final Image img) {
List lst = new ArrayList<>();
//TODO lst.add(new ExtractAction("Extract Image", img));
- lst.add(new ExtractUnallocAction(
+ try {
+ lst.add(new ExtractUnallocAction(
NbBundle.getMessage(this.getClass(), "ExplorerNodeActionVisitor.action.extUnallocToSingleFiles"), img));
+ } catch (NoCurrentCaseException ex) {
+ Logger.getLogger(ExplorerNodeActionVisitor.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
+ }
return lst;
}
@@ -85,7 +92,8 @@ public class ExplorerNodeActionVisitor extends ContentVisitor.Default visit(final Volume vol) {
List lst = new ArrayList<>();
lst.add(new ExtractUnallocAction(
- NbBundle.getMessage(this.getClass(), "ExplorerNodeActionVisitor.action.extUnallocToSingleFile"), vol));
+ NbBundle.getMessage(this.getClass(), "ExplorerNodeActionVisitor.action.extUnallocToSingleFile"), vol));
+
return lst;
}
diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java
index 1960089075..893bd94da4 100644
--- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java
+++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java
@@ -69,14 +69,20 @@ final class ExtractUnallocAction extends AbstractAction {
private long currentImage = 0L;
private final boolean isImage;
- public ExtractUnallocAction(String title, Volume volume) {
+ public ExtractUnallocAction(String title, Volume volume){
super(title);
isImage = false;
- OutputFileData outputFileData = new OutputFileData(volume);
- filesToExtract.add(outputFileData);
+ try {
+ OutputFileData outputFileData = new OutputFileData(volume);
+ filesToExtract.add(outputFileData);
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex);
+ setEnabled(false);
+ }
+
}
- public ExtractUnallocAction(String title, Image image) {
+ public ExtractUnallocAction(String title, Image image) throws NoCurrentCaseException {
super(title);
isImage = true;
currentImage = image.getId();
@@ -595,15 +601,17 @@ final class ExtractUnallocAction extends AbstractAction {
* Contingency constructor in event no VolumeSystem exists on an Image.
*
* @param img Image file to be analyzed
+ *
+ * @throws NoCurrentCaseException if there is no open case.
*/
- OutputFileData(Image img) {
+ OutputFileData(Image img) throws NoCurrentCaseException {
this.layoutFiles = getUnallocFiles(img);
Collections.sort(layoutFiles, new SortObjId());
this.volumeId = 0;
this.imageId = img.getId();
this.imageName = img.getName();
this.fileName = this.imageName + "-Unalloc-" + this.imageId + "-" + 0 + ".dat"; //NON-NLS
- this.fileInstance = new File(Case.getCurrentCase().getExportDirectory() + File.separator + this.fileName);
+ this.fileInstance = new File(Case.getOpenCase().getExportDirectory() + File.separator + this.fileName);
this.sizeInBytes = calcSizeInBytes();
}
@@ -611,8 +619,10 @@ final class ExtractUnallocAction extends AbstractAction {
* Default constructor for extracting info from Volumes.
*
* @param volume Volume file to be analyzed
+ *
+ * @throws NoCurrentCaseException if there is no open case.
*/
- OutputFileData(Volume volume) {
+ OutputFileData(Volume volume) throws NoCurrentCaseException {
try {
this.imageName = volume.getDataSource().getName();
this.imageId = volume.getDataSource().getId();
@@ -623,7 +633,7 @@ final class ExtractUnallocAction extends AbstractAction {
this.imageId = 0;
}
this.fileName = this.imageName + "-Unalloc-" + this.imageId + "-" + volumeId + ".dat"; //NON-NLS
- this.fileInstance = new File(Case.getCurrentCase().getExportDirectory() + File.separator + this.fileName);
+ this.fileInstance = new File(Case.getOpenCase().getExportDirectory() + File.separator + this.fileName);
this.layoutFiles = getUnallocFiles(volume);
Collections.sort(layoutFiles, new SortObjId());
this.sizeInBytes = calcSizeInBytes();
diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java
index a41acd1c58..ebdfb6146f 100644
--- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java
+++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java
@@ -518,7 +518,7 @@ final class DataSourceIngestJob {
*/
if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) {
logger.log(Level.INFO, "Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
- DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this, this.files);
+ DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this);
} else if (this.hasFirstStageDataSourceIngestPipeline()) {
logger.log(Level.INFO, "Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this);
@@ -827,7 +827,7 @@ final class DataSourceIngestJob {
}
/**
- * Adds more files from the data source for this job to the job, i.e., adds
+ * Adds more files from the data source for this job to the job, e.g., adds
* extracted or carved files. Not currently supported for the second stage
* of the job.
*
@@ -835,9 +835,7 @@ final class DataSourceIngestJob {
*/
void addFiles(List files) {
if (DataSourceIngestJob.Stages.FIRST == this.stage) {
- for (AbstractFile file : files) {
- DataSourceIngestJob.taskScheduler.scheduleFastTrackedFileIngestTask(this, file);
- }
+ DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this, files);
} else {
DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS
}
diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
index 65d0162d6d..218626985a 100644
--- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
+++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
@@ -121,7 +121,7 @@ public class IngestManager {
private final AtomicLong nextIngestManagerTaskId = new AtomicLong(0L);
private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-start-ingest-jobs-%d").build()); //NON-NLS;
private final Map> startIngestJobFutures = new ConcurrentHashMap<>();
- private final Map ingestJobsById = new ConcurrentHashMap<>();
+ private final Map ingestJobsById = new HashMap<>();
private final ExecutorService dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS;
private final ExecutorService fileLevelIngestJobTasksExecutor;
private final ExecutorService eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS;
@@ -399,13 +399,17 @@ public class IngestManager {
ingestMonitor.start();
}
- ingestJobsById.put(job.getId(), job);
+ synchronized (ingestJobsById) {
+ ingestJobsById.put(job.getId(), job);
+ }
errors = job.start();
if (errors.isEmpty()) {
this.fireIngestJobStarted(job.getId());
IngestManager.logger.log(Level.INFO, "Ingest job {0} started", job.getId()); //NON-NLS
} else {
- this.ingestJobsById.remove(job.getId());
+ synchronized (ingestJobsById) {
+ this.ingestJobsById.remove(job.getId());
+ }
for (IngestModuleError error : errors) {
logger.log(Level.SEVERE, String.format("Error starting %s ingest module for job %d", error.getModuleDisplayName(), job.getId()), error.getThrowable()); //NON-NLS
}
@@ -438,7 +442,9 @@ public class IngestManager {
*/
void finishIngestJob(IngestJob job) {
long jobId = job.getId();
- ingestJobsById.remove(jobId);
+ synchronized (ingestJobsById) {
+ ingestJobsById.remove(jobId);
+ }
if (!job.isCancelled()) {
IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); //NON-NLS
fireIngestJobCompleted(jobId);
@@ -455,7 +461,9 @@ public class IngestManager {
* @return True or false.
*/
public boolean isIngestRunning() {
- return !ingestJobsById.isEmpty();
+ synchronized (ingestJobsById) {
+ return !ingestJobsById.isEmpty();
+ }
}
/**
@@ -467,9 +475,11 @@ public class IngestManager {
startIngestJobFutures.values().forEach((handle) -> {
handle.cancel(true);
});
- this.ingestJobsById.values().forEach((job) -> {
- job.cancel(reason);
- });
+ synchronized (ingestJobsById) {
+ this.ingestJobsById.values().forEach((job) -> {
+ job.cancel(reason);
+ });
+ }
}
/**
@@ -770,9 +780,11 @@ public class IngestManager {
*/
List getIngestJobSnapshots() {
List snapShots = new ArrayList<>();
- ingestJobsById.values().forEach((job) -> {
- snapShots.addAll(job.getDataSourceIngestJobSnapshots());
- });
+ synchronized (ingestJobsById) {
+ ingestJobsById.values().forEach((job) -> {
+ snapShots.addAll(job.getDataSourceIngestJobSnapshots());
+ });
+ }
return snapShots;
}
@@ -808,7 +820,9 @@ public class IngestManager {
public Void call() {
try {
if (Thread.currentThread().isInterrupted()) {
- ingestJobsById.remove(job.getId());
+ synchronized (ingestJobsById) {
+ ingestJobsById.remove(job.getId());
+ }
return null;
}
diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java
index b9cee95687..36b09035b1 100644
--- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java
+++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java
@@ -21,12 +21,13 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
-import java.util.HashSet;
+import java.util.Deque;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
@@ -40,64 +41,20 @@ import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
- * Creates ingest tasks for ingest jobs, queuing the tasks in priority order for
- * execution by the ingest manager's ingest threads.
+ * Creates ingest tasks for data source ingest jobs, queueing the tasks in
+ * priority order for execution by the ingest manager's ingest threads.
*/
final class IngestTasksScheduler {
- private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName());
private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
+ private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName());
private static IngestTasksScheduler instance;
-
- /**
- * Scheduling of data source ingest tasks is accomplished by putting them in
- * a FIFO queue to be consumed by the ingest threads, so the queue is
- * wrapped in a "dispenser" that implements the IngestTaskQueue interface
- * and is exposed via a getter method.
- */
- private final LinkedBlockingQueue pendingDataSourceTasks;
- private final DataSourceIngestTaskQueue dataSourceTasksDispenser;
-
- /**
- * Scheduling of file ingest tasks is accomplished by "shuffling" them
- * through a sequence of internal queues that allows for the interleaving of
- * tasks from different ingest jobs based on priority. These scheduling
- * queues are:
- *
- * 1. Root directory tasks (priority queue)
- *
- * 2. Directory tasks (FIFO queue)
- *
- * 3. Pending file tasks (LIFO queue).
- *
- * The pending file tasks queue is LIFO to handle large numbers of files
- * extracted from archive files. At least one image has been processed that
- * had a folder full of archive files. The queue grew to have thousands of
- * entries, as each successive archive file was expanded, so now extracted
- * files get added to the front of the queue so that in such a scenario they
- * would be processed before the expansion of the next archive file.
- *
- * Tasks in the pending file tasks queue are ready to be consumed by the
- * ingest threads, so the queue is wrapped in a "dispenser" that implements
- * the IngestTaskQueue interface and is exposed via a getter method.
- */
- private final TreeSet rootDirectoryTasks;
- private final List directoryTasks;
- private final BlockingDeque pendingFileTasks;
- private final FileIngestTaskQueue fileTasksDispenser;
-
- /**
- * The ingest tasks scheduler allows ingest jobs to query it to see if there
- * are any tasks in progress for the job. To make this possible, the ingest
- * tasks scheduler needs to keep track not only of the tasks in its queues,
- * but also of the tasks that have been handed out for processing by the
- * ingest threads. Therefore all ingest tasks are added to this list when
- * they are created and are not removed when an ingest thread takes an
- * ingest task. Instead, the ingest thread calls back into the scheduler
- * when the task is completed, at which time the task will be removed from
- * this list.
- */
- private final Set tasksInProgress;
+ private final DataSourceIngestTaskQueue dataSourceTaskQueueForIngestThreads;
+ private final List queuedAndRunningDataSourceTasks;
+ private final TreeSet rootFileTaskQueue;
+ private final Deque directoryFileTaskQueue;
+ private final FileIngestTaskQueue fileTaskQueueForIngestThreads;
+ private final List queuedAndRunningFileTasks;
/**
* Gets the ingest tasks scheduler singleton.
@@ -113,52 +70,53 @@ final class IngestTasksScheduler {
* Constructs an ingest tasks scheduler.
*/
private IngestTasksScheduler() {
- this.pendingDataSourceTasks = new LinkedBlockingQueue<>();
- this.dataSourceTasksDispenser = new DataSourceIngestTaskQueue();
- this.rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator());
- this.directoryTasks = new ArrayList<>();
- this.pendingFileTasks = new LinkedBlockingDeque<>();
- this.fileTasksDispenser = new FileIngestTaskQueue();
- this.tasksInProgress = new HashSet<>();
+ this.queuedAndRunningDataSourceTasks = new LinkedList<>();
+ this.dataSourceTaskQueueForIngestThreads = new DataSourceIngestTaskQueue();
+ this.rootFileTaskQueue = new TreeSet<>(new RootDirectoryTaskComparator());
+ this.directoryFileTaskQueue = new LinkedList<>();
+ this.queuedAndRunningFileTasks = new LinkedList<>();
+ this.fileTaskQueueForIngestThreads = new FileIngestTaskQueue();
}
/**
- * Gets this ingest task scheduler's implementation of the IngestTaskQueue
- * interface for data source ingest tasks.
+ * Gets the data source level ingest tasks queue. This queue is a blocking
+ * queue intended for use by the ingest manager's data source ingest
+ * threads.
*
- * @return The data source ingest tasks queue.
+ * @return The queue.
*/
IngestTaskQueue getDataSourceIngestTaskQueue() {
- return this.dataSourceTasksDispenser;
+ return this.dataSourceTaskQueueForIngestThreads;
}
/**
- * Gets this ingest task scheduler's implementation of the IngestTaskQueue
- * interface for file ingest tasks.
+ * Gets the file level ingest tasks queue. This queue is a blocking queue
+ * intended for use by the ingest manager's file ingest threads.
*
- * @return The file ingest tasks queue.
+ * @return The queue.
*/
IngestTaskQueue getFileIngestTaskQueue() {
- return this.fileTasksDispenser;
+ return this.fileTaskQueueForIngestThreads;
}
/**
* Schedules a data source level ingest task and file level ingest tasks for
- * an ingest job. Either all of the files in the data source or a given
- * subset of the files will be scheduled.
+ * a data source ingest job.
*
- * @param job The data source ingest job.
- * @param files A subset of the files for the data source.
+ * @param job The data source ingest job.
*/
- synchronized void scheduleIngestTasks(DataSourceIngestJob job, List files) {
+ synchronized void scheduleIngestTasks(DataSourceIngestJob job) {
if (!job.isCancelled()) {
- // Scheduling of both a data source ingest task and file ingest tasks
- // for a job must be an atomic operation. Otherwise, the data source
- // task might be completed before the file tasks are scheduled,
- // resulting in a potential false positive when another thread checks
- // whether or not all the tasks for the job are completed.
+ /*
+ * Scheduling of both the data source ingest task and the initial
+ * file ingest tasks for a job must be an atomic operation.
+ * Otherwise, the data source task might be completed before the
+ * file tasks are scheduled, resulting in a potential false positive
+ * when another thread checks whether or not all the tasks for the
+ * job are completed.
+ */
this.scheduleDataSourceIngestTask(job);
- this.scheduleFileIngestTasks(job, files);
+ this.scheduleFileIngestTasks(job);
}
}
@@ -170,41 +128,29 @@ final class IngestTasksScheduler {
synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) {
if (!job.isCancelled()) {
DataSourceIngestTask task = new DataSourceIngestTask(job);
- this.tasksInProgress.add(task);
+ this.queuedAndRunningDataSourceTasks.add(task);
try {
- this.pendingDataSourceTasks.put(task);
+ this.dataSourceTaskQueueForIngestThreads.add(task);
} catch (InterruptedException ex) {
- /**
- * The current thread was interrupted while blocked on a full
- * queue. Discard the task and reset the interrupted flag.
- */
- this.tasksInProgress.remove(task);
+ IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while a data source ingest thread was blocked on a full queue", ex);
+ this.queuedAndRunningDataSourceTasks.remove(task);
Thread.currentThread().interrupt();
}
}
}
/**
- * Schedules file level ingest tasks for a data source ingest job. Either
- * all of the files in the data source or a given subset of the files will
- * be scheduled.
+ * Schedules file level ingest tasks for a data source ingest job.
*
- * @param job The data source ingest job.
- * @param files A subset of the files for the data source.
+ * @param job The data source ingest job.
*/
- synchronized void scheduleFileIngestTasks(DataSourceIngestJob job, List files) {
+ synchronized void scheduleFileIngestTasks(DataSourceIngestJob job) {
if (!job.isCancelled()) {
- List candidateFiles = new ArrayList<>();
- if (files.isEmpty()) {
- getTopLevelFiles(job.getDataSource(), candidateFiles);
- } else {
- candidateFiles.addAll(files);
- }
- for (AbstractFile firstLevelFile : candidateFiles) {
- FileIngestTask task = new FileIngestTask(job, firstLevelFile);
+ List candidateFiles = getTopLevelFiles(job.getDataSource());
+ for (AbstractFile file : candidateFiles) {
+ FileIngestTask task = new FileIngestTask(job, file);
if (IngestTasksScheduler.shouldEnqueueFileTask(task)) {
- this.tasksInProgress.add(task);
- this.rootDirectoryTasks.add(task);
+ this.rootFileTaskQueue.add(task);
}
}
shuffleFileTaskQueues();
@@ -212,73 +158,119 @@ final class IngestTasksScheduler {
}
/**
- * Schedules a file ingest task for a data source ingest job. The task that
- * is created is added directly to the pending file tasks queues, i.e., it
- * is "fast tracked."
+ * Schedules file level ingest tasks for a subset of the files for a data
+ * source ingest job.
*
- * @param job The data source ingest job.
- * @param file A file.
+ * @param job The data source ingest job.
+ * @param files A subset of the files for the data source.
*/
- synchronized void scheduleFastTrackedFileIngestTask(DataSourceIngestJob job, AbstractFile file) {
+ synchronized void scheduleFileIngestTasks(DataSourceIngestJob job, Collection files) {
if (!job.isCancelled()) {
- FileIngestTask task = new FileIngestTask(job, file);
- if (IngestTasksScheduler.shouldEnqueueFileTask(task)) {
- this.tasksInProgress.add(task);
- addToPendingFileTasksQueue(task);
+ List newTasksForFileIngestThreads = new LinkedList<>();
+ for (AbstractFile file : files) {
+ /*
+ * Put the file directly into the queue for the file ingest
+ * threads, if it passes the filter for the job. The file is
+ * added to the queue for the ingest threads BEFORE the other
+ * queued tasks because the primary use case for this method is
+ * adding derived files from a higher priority task that
+ * preceded the tasks currently in the queue.
+ */
+ FileIngestTask task = new FileIngestTask(job, file);
+ if (shouldEnqueueFileTask(task)) {
+ newTasksForFileIngestThreads.add(task);
+ }
+
+ /*
+ * If the file or directory that was just queued has children,
+ * try to queue tasks for the children. Each child task will go
+ * into either the directory queue if it is a directory, or
+ * directly into the queue for the file ingest threads, if it
+ * passes the filter for the job.
+ */
+ try {
+ for (Content child : file.getChildren()) {
+ if (child instanceof AbstractFile) {
+ AbstractFile childFile = (AbstractFile) child;
+ FileIngestTask childTask = new FileIngestTask(job, childFile);
+ if (childFile.hasChildren()) {
+ this.directoryFileTaskQueue.add(childTask);
+ } else if (shouldEnqueueFileTask(childTask)) {
+ newTasksForFileIngestThreads.add(task);
+ }
+ }
+ }
+ } catch (TskCoreException ex) {
+ logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", file.getName(), file.getId()), ex); //NON-NLS
+ }
+ }
+
+ /*
+ * The files are added to the queue for the ingest threads BEFORE
+ * the other queued tasks because the primary use case for this
+ * method is adding derived files from a higher priority task that
+ * preceded the tasks currently in the queue.
+ */
+ for (FileIngestTask newTask : newTasksForFileIngestThreads) {
+ try {
+ this.queuedAndRunningFileTasks.add(newTask);
+ this.fileTaskQueueForIngestThreads.addFirst(newTask);
+ } catch (InterruptedException ex) {
+ this.queuedAndRunningFileTasks.remove(newTask);
+ IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while blocked on a full file ingest threads queue", ex);
+ Thread.currentThread().interrupt();
+ break;
+ }
}
}
}
/**
- * Allows an ingest thread to notify this ingest task scheduler that a task
- * has been completed.
+ * Allows an ingest thread to notify this ingest task scheduler that a data
+ * source level task has been completed.
*
* @param task The completed task.
*/
- synchronized void notifyTaskCompleted(IngestTask task) {
- tasksInProgress.remove(task);
+ synchronized void notifyTaskCompleted(DataSourceIngestTask task) {
+ this.queuedAndRunningDataSourceTasks.remove(task);
}
/**
- * Queries the task scheduler to determine whether or not all current ingest
- * tasks for an ingest job are completed.
+ * Allows an ingest thread to notify this ingest task scheduler that a file
+ * level task has been completed.
*
- * @param job The job for which the query is to be performed.
+ * @param task The completed task.
+ */
+ synchronized void notifyTaskCompleted(FileIngestTask task) {
+ this.queuedAndRunningFileTasks.remove(task);
+ shuffleFileTaskQueues();
+ }
+
+ /**
+ * Queries the task scheduler to determine whether or not all of the ingest
+ * tasks for a data source ingest job have been completed.
+ *
+ * @param job The data source ingest job.
*
* @return True or false.
*/
synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) {
- for (IngestTask task : tasksInProgress) {
- if (task.getIngestJob().getId() == job.getId()) {
- return false;
- }
- }
- return true;
+ return !hasTasksForJob(this.queuedAndRunningDataSourceTasks, job)
+ && !hasTasksForJob(this.rootFileTaskQueue, job)
+ && !hasTasksForJob(this.directoryFileTaskQueue, job)
+ && !hasTasksForJob(this.queuedAndRunningFileTasks, job);
}
/**
- * Clears the "upstream" task scheduling queues for an ingest job, but does
- * nothing about tasks that have already been shuffled into the concurrently
- * accessed blocking queues shared with the ingest threads. Note that tasks
- * in the "downstream" queues or already taken by the ingest threads will be
- * flushed out when the ingest threads call back with their task completed
- * notifications.
+ * Clears the "upstream" task scheduling queues for a data source ingest
+ * job, but does nothing about tasks that have already been moved into the
+ * queue that is consumed by the file ingest threads.
*
- * @param job The job for which the tasks are to to canceled.
+ * @param job The data source ingest job.
*/
synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) {
- /**
- * This code should not flush the blocking queues that are concurrently
- * accessed by the ingest threads. This is because the "lock striping"
- * and "weakly consistent" iterators of these collections make it so
- * that this code could have a different view of the queues than the
- * ingest threads. It does clean out the directory level tasks before
- * they are exploded into file tasks.
- */
- long jobId = job.getId();
- this.removeTasksForJob(this.rootDirectoryTasks, jobId);
- this.removeTasksForJob(this.directoryTasks, jobId);
- this.shuffleFileTaskQueues();
+ this.removeTasksForJob(this.rootFileTaskQueue, job);
+ this.removeTasksForJob(this.directoryFileTaskQueue, job);
}
/**
@@ -286,10 +278,12 @@ final class IngestTasksScheduler {
* files and virtual directories for a data source. Used to create file
* tasks to put into the root directories queue.
*
- * @param dataSource The data source.
- * @param topLevelFiles The top level files are added to this list.
+ * @param dataSource The data source.
+ *
+ * @return The top level files.
*/
- private static void getTopLevelFiles(Content dataSource, List topLevelFiles) {
+ private static List getTopLevelFiles(Content dataSource) {
+ List topLevelFiles = new ArrayList<>();
Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
// The data source is itself a file to be processed.
@@ -317,74 +311,113 @@ final class IngestTasksScheduler {
}
}
}
+ return topLevelFiles;
}
/**
- * "Shuffles" the file task queues to ensure that there is at least one task
- * in the pending file ingest tasks queue, as long as there are still file
- * ingest tasks to be performed.
+ * Schedules file ingest tasks for the ingest manager's file ingest threads
+ * by "shuffling" them through a sequence of three queues that allows for
+ * the interleaving of tasks from different data source ingest jobs based on
+ * priority. The sequence of queues is:
+ *
+ * 1. The root file tasks priority queue, which contains file tasks for the
+ * root objects of the data sources that are being analyzed. For example,
+ * the root tasks for a disk image data source are typically the tasks for
+ * the contents of the root directories of the file systems. This queue is a
+ * priority queue that attempts to ensure that user directory content is
+ * analyzed before general file system content. It feeds into the directory
+ * tasks queue.
+ *
+ * 2. The directory file tasks queue, which contains root file tasks
+ * shuffled out of the root tasks queue, plus directory tasks discovered in
+ * the descent from the root tasks to the final leaf tasks in the content
+ * trees that are being analyzed for the data source ingest jobs. This queue
+ * is a FIFO queue. It feeds into the file tasks queue for the ingest
+ * manager's file ingest threads.
+ *
+ * 3. The file tasks queue for the ingest manager's file ingest threads.
+ * This queue is a blocking deque that is FIFO during a shuffle to maintain
+ * task prioritization, but LIFO when adding derived files to it directly
+ * during ingest. The reason for the LIFO additions is to give priority
+ * derived files of priority files.
+ *
+ * There is a fourth collection of file tasks, a "tracking" list, that keeps
+ * track of the file tasks that are either in the tasks queue for the file
+ * ingest threads, or are in the process of being analyzed in a file ingest
+ * thread. This queue is vital to the ingest task scheduler's ability to
+ * determine when all of the ingest tasks for a data source ingest job have
+ * been completed. It is also used to drive this shuffling algorithm -
+ * whenever this list is empty, the two "upstream" queues are "shuffled" to
+ * queue more tasks for the file ingest threads.
*/
synchronized private void shuffleFileTaskQueues() {
- // This is synchronized because it is called both by synchronized
- // methods of this ingest scheduler and an unsynchronized method of its
- // file tasks "dispenser".
- while (true) {
- // Loop until either the pending file tasks queue is NOT empty
- // or the upstream queues that feed into it ARE empty.
- if (!this.pendingFileTasks.isEmpty()) {
- // There are file tasks ready to be consumed, exit.
- return;
- }
- if (this.directoryTasks.isEmpty()) {
- if (this.rootDirectoryTasks.isEmpty()) {
- // There are no root directory tasks to move into the
- // directory queue, exit.
- return;
+ List newTasksForFileIngestThreads = new LinkedList<>();
+ while (this.queuedAndRunningFileTasks.isEmpty()) {
+ /*
+ * If the directory file task queue is empty, move the highest
+ * priority root file task, if there is one, into it. If both the
+ * root and the directory file task queues are empty, there is
+ * nothing left to shuffle, so exit.
+ */
+ if (this.directoryFileTaskQueue.isEmpty()) {
+ if (!this.rootFileTaskQueue.isEmpty()) {
+ this.directoryFileTaskQueue.add(this.rootFileTaskQueue.pollFirst());
} else {
- // Move the next root directory task into the
- // directories queue. Note that the task was already
- // added to the tasks in progress list when the task was
- // created in scheduleFileIngestTasks().
- this.directoryTasks.add(this.rootDirectoryTasks.pollFirst());
+ return;
}
}
- // Try to add the most recently added directory from the
- // directory tasks queue to the pending file tasks queue.
- FileIngestTask directoryTask = this.directoryTasks.remove(this.directoryTasks.size() - 1);
+ /*
+ * Try to move the next task from the directory task queue into the
+ * queue for the file ingest threads, if it passes the filter for
+ * the job. The file is added to the queue for the ingest threads
+ * AFTER the higher priority tasks that preceded it.
+ */
+ final FileIngestTask directoryTask = this.directoryFileTaskQueue.pollLast();
if (shouldEnqueueFileTask(directoryTask)) {
- addToPendingFileTasksQueue(directoryTask);
- } else {
- this.tasksInProgress.remove(directoryTask);
+ newTasksForFileIngestThreads.add(directoryTask);
+ this.queuedAndRunningFileTasks.add(directoryTask);
}
- // If the directory contains subdirectories or files, try to
- // enqueue tasks for them as well.
+ /*
+ * If the directory (or root level file) that was just queued has
+ * children, try to queue tasks for the children. Each child task
+ * will go into either the directory queue if it is a directory, or
+ * into the queue for the file ingest threads, if it passes the
+ * filter for the job. The file is added to the queue for the ingest
+ * threads AFTER the higher priority tasks that preceded it.
+ */
final AbstractFile directory = directoryTask.getFile();
try {
for (Content child : directory.getChildren()) {
if (child instanceof AbstractFile) {
- AbstractFile file = (AbstractFile) child;
- FileIngestTask childTask = new FileIngestTask(directoryTask.getIngestJob(), file);
- if (file.hasChildren()) {
- // Found a subdirectory, put the task in the
- // pending directory tasks queue. Note the
- // addition of the task to the tasks in progress
- // list. This is necessary because this is the
- // first appearance of this task in the queues.
- this.tasksInProgress.add(childTask);
- this.directoryTasks.add(childTask);
+ AbstractFile childFile = (AbstractFile) child;
+ FileIngestTask childTask = new FileIngestTask(directoryTask.getIngestJob(), childFile);
+ if (childFile.hasChildren()) {
+ this.directoryFileTaskQueue.add(childTask);
} else if (shouldEnqueueFileTask(childTask)) {
- // Found a file, put the task directly into the
- // pending file tasks queue.
- this.tasksInProgress.add(childTask);
- addToPendingFileTasksQueue(childTask);
+ newTasksForFileIngestThreads.add(childTask);
+ this.queuedAndRunningFileTasks.add(childTask);
}
}
}
} catch (TskCoreException ex) {
- String errorMessage = String.format("An error occurred getting the children of %s", directory.getName()); //NON-NLS
- logger.log(Level.SEVERE, errorMessage, ex);
+ logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", directory.getName(), directory.getId()), ex); //NON-NLS
+ }
+ }
+
+ /*
+ * The files are added to the queue for the ingest threads AFTER the
+ * higher priority tasks that preceded them.
+ */
+ for (FileIngestTask newTask : newTasksForFileIngestThreads) {
+ try {
+ this.fileTaskQueueForIngestThreads.addFirst(newTask);
+ } catch (InterruptedException ex) {
+ this.queuedAndRunningFileTasks.remove(newTask);
+ IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while blocked on a full file ingest threads queue", ex);
+ Thread.currentThread().interrupt();
+ break;
}
}
}
@@ -463,44 +496,44 @@ final class IngestTasksScheduler {
}
/**
- * Adds a file ingest task to the blocking pending tasks queue.
+ * Checks whether or not a collection of ingest tasks includes a task for a
+ * given data source ingest job.
*
- * @param task The task to add.
+ * @param tasks The tasks.
+ * @param job The data source ingest job.
+ *
+ * @return True if there are no tasks for the job, false otherwise.
*/
- synchronized private void addToPendingFileTasksQueue(FileIngestTask task) {
- try {
- this.pendingFileTasks.putFirst(task);
- } catch (InterruptedException ex) {
- /**
- * The current thread was interrupted while blocked on a full queue.
- * Discard the task and reset the interrupted flag.
- */
- this.tasksInProgress.remove(task);
- Thread.currentThread().interrupt();
+ synchronized private boolean hasTasksForJob(Collection tasks, DataSourceIngestJob job) {
+ long jobId = job.getId();
+ for (IngestTask task : tasks) {
+ if (task.getIngestJob().getId() == jobId) {
+ return true;
+ }
}
+ return false;
}
/**
- * Removes all of the ingest tasks associated with an ingest job from a
- * tasks queue. The task is removed from the the tasks in progress list as
- * well.
+ * Removes all of the ingest tasks associated with a data source ingest job
+ * from a tasks collection.
*
- * @param taskQueue The queue from which to remove the tasks.
- * @param jobId The id of the job for which the tasks are to be removed.
+ * @param tasks The collection from which to remove the tasks.
+ * @param job THe data source ingest job.
*/
- synchronized private void removeTasksForJob(Collection taskQueue, long jobId) {
- Iterator iterator = taskQueue.iterator();
+ synchronized private void removeTasksForJob(Collection tasks, DataSourceIngestJob job) {
+ long jobId = job.getId();
+ Iterator iterator = tasks.iterator();
while (iterator.hasNext()) {
IngestTask task = iterator.next();
if (task.getIngestJob().getId() == jobId) {
- this.tasksInProgress.remove(task);
iterator.remove();
}
}
}
/**
- * Counts the number of ingest tasks in a task queue for a given job.
+ * Counts the number of ingest tasks in a tasks collection for a given job.
*
* @param queue The queue for which to count tasks.
* @param jobId The id of the job for which the tasks are to be counted.
@@ -511,7 +544,7 @@ final class IngestTasksScheduler {
Iterator iterator = queue.iterator();
int count = 0;
while (iterator.hasNext()) {
- IngestTask task = (IngestTask) iterator.next();
+ IngestTask task = iterator.next();
if (task.getIngestJob().getId() == jobId) {
count++;
}
@@ -549,8 +582,15 @@ final class IngestTasksScheduler {
}
}
+ /**
+ * Used to prioritize file ingest tasks in the root tasks queue so that
+ * user content is processed first.
+ */
private static class AbstractFilePriority {
+ private AbstractFilePriority() {
+ }
+
enum Priority {
LAST, LOW, MEDIUM, HIGH
@@ -642,28 +682,43 @@ final class IngestTasksScheduler {
}
/**
- * Wraps access to pending data source ingest tasks in the interface
- * required by the ingest threads.
+ * A blocking queue of data source ingest tasks for the ingest manager's
+ * data source ingest threads.
*/
private final class DataSourceIngestTaskQueue implements IngestTaskQueue {
+ private final BlockingQueue tasks = new LinkedBlockingQueue<>();
+
+ private void add(DataSourceIngestTask task) throws InterruptedException {
+ this.tasks.put(task);
+ }
+
@Override
public IngestTask getNextTask() throws InterruptedException {
- return IngestTasksScheduler.this.pendingDataSourceTasks.take();
+ return tasks.take();
}
+
}
/**
- * Wraps access to pending file ingest tasks in the interface required by
- * the ingest threads.
+ * A blocking, LIFO queue of data source ingest tasks for the ingest
+ * manager's data source ingest threads.
*/
private final class FileIngestTaskQueue implements IngestTaskQueue {
+ private final BlockingDeque tasks = new LinkedBlockingDeque<>();
+
+ private void addFirst(FileIngestTask task) throws InterruptedException {
+ this.tasks.putFirst(task);
+ }
+
+ private void addLast(FileIngestTask task) throws InterruptedException {
+ this.tasks.putLast(task);
+ }
+
@Override
public IngestTask getNextTask() throws InterruptedException {
- FileIngestTask task = IngestTasksScheduler.this.pendingFileTasks.takeFirst();
- shuffleFileTaskQueues();
- return task;
+ return tasks.takeFirst();
}
}
@@ -674,10 +729,10 @@ final class IngestTasksScheduler {
class IngestJobTasksSnapshot {
private final long jobId;
+ private final long dsQueueSize;
private final long rootQueueSize;
private final long dirQueueSize;
private final long fileQueueSize;
- private final long dsQueueSize;
private final long runningListSize;
/**
@@ -687,11 +742,11 @@ final class IngestTasksScheduler {
*/
IngestJobTasksSnapshot(long jobId) {
this.jobId = jobId;
- this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootDirectoryTasks, jobId);
- this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.directoryTasks, jobId);
- this.fileQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingFileTasks, jobId);
- this.dsQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingDataSourceTasks, jobId);
- this.runningListSize = countTasksForJob(IngestTasksScheduler.this.tasksInProgress, jobId);
+ this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootFileTaskQueue, jobId);
+ this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.directoryFileTaskQueue, jobId);
+ this.fileQueueSize = countTasksForJob(IngestTasksScheduler.this.fileTaskQueueForIngestThreads.tasks, jobId);
+ this.dsQueueSize = countTasksForJob(IngestTasksScheduler.this.dataSourceTaskQueueForIngestThreads.tasks, jobId);
+ this.runningListSize = countTasksForJob(IngestTasksScheduler.this.queuedAndRunningDataSourceTasks, jobId) + countTasksForJob(IngestTasksScheduler.this.queuedAndRunningFileTasks, jobId);
}
/**
diff --git a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java
index d322d1026f..4698b68006 100644
--- a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java
+++ b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java
@@ -93,7 +93,7 @@ public final class RunIngestModulesAction extends AbstractAction {
* Constructs an action that invokes the Run Ingest Modules wizard for the
* children of a file.
*
- * @param file The file.
+ * @param parentFile The file.
*/
public RunIngestModulesAction(AbstractFile parentFile) {
this.putValue(Action.NAME, Bundle.RunIngestModulesAction_name());
diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java
index 0dbb0300f2..0f8b009c52 100644
--- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java
+++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java
@@ -547,7 +547,7 @@ public final class FilesSet implements Serializable {
/**
* Construct a meta-type condition.
*
- * @param metaType The meta-type to match, must.
+ * @param type The meta-type to match, must.
*/
public MetaTypeCondition(Type type) {
this.type = type;
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java
index ea5b7b5ee3..d097882053 100644
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java
@@ -38,7 +38,11 @@ import java.util.logging.Level;
import javax.swing.DefaultListSelectionModel;
import java.awt.Color;
import java.beans.PropertyChangeEvent;
+import java.beans.PropertyChangeListener;
import java.io.File;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Logger;
import javax.swing.JOptionPane;
import javax.swing.JPanel;
@@ -149,6 +153,11 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
private Color pendingTableBackground;
private Color pendingTablelForeground;
+ /**
+ * Maintain a mapping of each service to it's last status update.
+ */
+ private final ConcurrentHashMap statusByService;
+
/*
* The enum is used in conjunction with the DefaultTableModel class to
* provide table models for the JTables used to display a view of the
@@ -235,6 +244,8 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
* controlling automated ingest for a single node within the cluster.
*/
private AutoIngestControlPanel() {
+
+ this.statusByService = new ConcurrentHashMap<>();
//Disable the main window so they can only use the dashboard (if we used setVisible the taskBar icon would go away)
WindowManager.getDefault().getMainWindow().setEnabled(false);
@@ -248,6 +259,9 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
completedTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0);
initComponents(); // Generated code.
+ statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down"));
+ statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down"));
+ statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down"));
setServicesStatusMessage();
initPendingJobsTable();
initRunningJobsTable();
@@ -260,6 +274,25 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
UIManager.put("PopupMenu.consumeEventOnClose", false);
}
+ /**
+ * Update status of the services on the dashboard
+ */
+ private void displayServicesStatus() {
+ tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message",
+ statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()),
+ statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()),
+ statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()),
+ statusByService.get(ServicesMonitor.Service.MESSAGING.toString())));
+ String upStatus = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Up");
+ if (statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()).compareTo(upStatus) != 0
+ || statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()).compareTo(upStatus) != 0
+ || statusByService.get(ServicesMonitor.Service.MESSAGING.toString()).compareTo(upStatus) != 0) {
+ tbServicesStatusMessage.setForeground(Color.RED);
+ } else {
+ tbServicesStatusMessage.setForeground(Color.BLACK);
+ }
+ }
+
/**
* Queries the services monitor and sets the text for the services status
* text box.
@@ -274,15 +307,11 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
private void setServicesStatusMessage() {
new SwingWorker() {
- String caseDatabaseServerStatus = ServicesMonitor.ServiceStatus.DOWN.toString();
- String keywordSearchServiceStatus = ServicesMonitor.ServiceStatus.DOWN.toString();
- String messagingStatus = ServicesMonitor.ServiceStatus.DOWN.toString();
-
@Override
protected Void doInBackground() throws Exception {
- caseDatabaseServerStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE);
- keywordSearchServiceStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH);
- messagingStatus = getServiceStatus(ServicesMonitor.Service.MESSAGING);
+ statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE));
+ statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH));
+ statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), getServiceStatus(ServicesMonitor.Service.MESSAGING));
return null;
}
@@ -311,15 +340,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
@Override
protected void done() {
- tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message", caseDatabaseServerStatus, keywordSearchServiceStatus, keywordSearchServiceStatus, messagingStatus));
- String upStatus = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Up");
- if (caseDatabaseServerStatus.compareTo(upStatus) != 0
- || keywordSearchServiceStatus.compareTo(upStatus) != 0
- || messagingStatus.compareTo(upStatus) != 0) {
- tbServicesStatusMessage.setForeground(Color.RED);
- } else {
- tbServicesStatusMessage.setForeground(Color.BLACK);
- }
+ displayServicesStatus();
}
}.execute();
@@ -682,12 +703,33 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
return;
}
- /*
- * Subscribe to services monitor events.
- */
- ServicesMonitor.getInstance().addSubscriber((PropertyChangeEvent evt) -> {
- setServicesStatusMessage();
- });
+ PropertyChangeListener propChangeListener = (PropertyChangeEvent evt) -> {
+
+ String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).toString();
+ String status = evt.getNewValue().toString();
+
+ if (status.equals(ServicesMonitor.ServiceStatus.UP.toString())) {
+ status = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Up");
+ } else if (status.equals(ServicesMonitor.ServiceStatus.DOWN.toString())) {
+ status = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down");
+ SYS_LOGGER.log(Level.SEVERE, "Connection to {0} is down", serviceDisplayName); //NON-NLS
+ }
+
+ // if the status update is for an existing service who's status hasn't changed - do nothing.
+ if (statusByService.containsKey(serviceDisplayName) && status.equals(statusByService.get(serviceDisplayName))) {
+ return;
+ }
+
+ statusByService.put(serviceDisplayName, status);
+ displayServicesStatus();
+ };
+
+ // Subscribe to all multi-user services in order to display their status
+ Set servicesList = new HashSet<>();
+ servicesList.add(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString());
+ servicesList.add(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString());
+ servicesList.add(ServicesMonitor.Service.MESSAGING.toString());
+ ServicesMonitor.getInstance().addSubscriber(servicesList, propChangeListener);
/*
* Register with the AIM as an observer.
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java
index 9324f782a3..3389a64a03 100644
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java
@@ -31,6 +31,10 @@ import java.util.logging.Level;
import javax.swing.DefaultListSelectionModel;
import java.awt.Color;
import java.beans.PropertyChangeEvent;
+import java.beans.PropertyChangeListener;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import javax.swing.JPanel;
import javax.swing.JTable;
import javax.swing.SwingWorker;
@@ -82,6 +86,11 @@ final class AutoIngestDashboard extends JPanel implements Observer {
private final DefaultTableModel runningTableModel;
private final DefaultTableModel completedTableModel;
private AutoIngestMonitor autoIngestMonitor;
+
+ /**
+ * Maintain a mapping of each service to it's last status update.
+ */
+ private final ConcurrentHashMap statusByService;
/**
* Creates a dashboard for monitoring an automated ingest cluster.
@@ -105,6 +114,8 @@ final class AutoIngestDashboard extends JPanel implements Observer {
* Constructs a panel for monitoring an automated ingest cluster.
*/
private AutoIngestDashboard() {
+ this.statusByService = new ConcurrentHashMap<>();
+
pendingTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0);
runningTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0);
@@ -112,6 +123,9 @@ final class AutoIngestDashboard extends JPanel implements Observer {
completedTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0);
initComponents();
+ statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down"));
+ statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down"));
+ statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down"));
setServicesStatusMessage();
initPendingJobsTable();
initRunningJobsTable();
@@ -122,6 +136,25 @@ final class AutoIngestDashboard extends JPanel implements Observer {
*/
UIManager.put("PopupMenu.consumeEventOnClose", false);
}
+
+ /**
+ * Update status of the services on the dashboard
+ */
+ private void displayServicesStatus() {
+ tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message",
+ statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()),
+ statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()),
+ statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()),
+ statusByService.get(ServicesMonitor.Service.MESSAGING.toString())));
+ String upStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up");
+ if (statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()).compareTo(upStatus) != 0
+ || statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()).compareTo(upStatus) != 0
+ || statusByService.get(ServicesMonitor.Service.MESSAGING.toString()).compareTo(upStatus) != 0) {
+ tbServicesStatusMessage.setForeground(Color.RED);
+ } else {
+ tbServicesStatusMessage.setForeground(Color.BLACK);
+ }
+ }
/**
* Queries the services monitor and sets the text for the services status
@@ -129,15 +162,12 @@ final class AutoIngestDashboard extends JPanel implements Observer {
*/
private void setServicesStatusMessage() {
new SwingWorker() {
- String caseDatabaseServerStatus = ServicesMonitor.ServiceStatus.DOWN.toString();
- String keywordSearchServiceStatus = ServicesMonitor.ServiceStatus.DOWN.toString();
- String messagingStatus = ServicesMonitor.ServiceStatus.DOWN.toString();
-
+
@Override
protected Void doInBackground() throws Exception {
- caseDatabaseServerStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE);
- keywordSearchServiceStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH);
- messagingStatus = getServiceStatus(ServicesMonitor.Service.MESSAGING);
+ statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE));
+ statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH));
+ statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), getServiceStatus(ServicesMonitor.Service.MESSAGING));
return null;
}
@@ -166,15 +196,7 @@ final class AutoIngestDashboard extends JPanel implements Observer {
@Override
protected void done() {
- tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message", caseDatabaseServerStatus, keywordSearchServiceStatus, keywordSearchServiceStatus, messagingStatus));
- String upStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up");
- if (caseDatabaseServerStatus.compareTo(upStatus) != 0
- || keywordSearchServiceStatus.compareTo(upStatus) != 0
- || messagingStatus.compareTo(upStatus) != 0) {
- tbServicesStatusMessage.setForeground(Color.RED);
- } else {
- tbServicesStatusMessage.setForeground(Color.BLACK);
- }
+ displayServicesStatus();
}
}.execute();
@@ -413,10 +435,38 @@ final class AutoIngestDashboard extends JPanel implements Observer {
* auto ingest job tables.
*/
private void startUp() throws AutoIngestMonitor.AutoIngestMonitorException {
- setServicesStatusMessage();
- ServicesMonitor.getInstance().addSubscriber((PropertyChangeEvent evt) -> {
- setServicesStatusMessage();
- });
+
+ PropertyChangeListener propChangeListener = (PropertyChangeEvent evt) -> {
+
+ String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).toString();
+ String status = evt.getNewValue().toString();
+
+ if (status.equals(ServicesMonitor.ServiceStatus.UP.toString())) {
+ status = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up");
+ LOGGER.log(Level.INFO, "Connection to {0} is up", serviceDisplayName); //NON-NLS
+ } else if (status.equals(ServicesMonitor.ServiceStatus.DOWN.toString())) {
+ status = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down");
+ LOGGER.log(Level.SEVERE, "Connection to {0} is down", serviceDisplayName); //NON-NLS
+ } else {
+ LOGGER.log(Level.INFO, "Status for {0} is {1}", new Object[]{serviceDisplayName, status}); //NON-NLS
+ }
+
+ // if the status update is for an existing service who's status hasn't changed - do nothing.
+ if (statusByService.containsKey(serviceDisplayName) && status.equals(statusByService.get(serviceDisplayName))) {
+ return;
+ }
+
+ statusByService.put(serviceDisplayName, status);
+ displayServicesStatus();
+ };
+
+ // Subscribe to all multi-user services in order to display their status
+ Set servicesList = new HashSet<>();
+ servicesList.add(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString());
+ servicesList.add(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString());
+ servicesList.add(ServicesMonitor.Service.MESSAGING.toString());
+ ServicesMonitor.getInstance().addSubscriber(servicesList, propChangeListener);
+
autoIngestMonitor = new AutoIngestMonitor();
autoIngestMonitor.addObserver(this);
autoIngestMonitor.startUp();
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
index 23380b15ae..416f2b394f 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
@@ -874,7 +874,7 @@ public class Server {
* if this does not exist then no server is recorded.
*
* Format of solrServerList.txt:
- * ,
+ * (host),(port)
* Ex: 10.1.2.34,8983
*
* @param rootOutputDirectory
diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
index 4cb6eaf8e3..8ae8dc0261 100644
--- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
+++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
@@ -41,17 +41,32 @@ import org.sleuthkit.datamodel.*;
abstract class Extract {
- protected Case currentCase = Case.getCurrentCase();
- protected SleuthkitCase tskCase = currentCase.getSleuthkitCase();
+ protected Case currentCase;
+ protected SleuthkitCase tskCase;
private final Logger logger = Logger.getLogger(this.getClass().getName());
private final ArrayList errorMessages = new ArrayList<>();
String moduleName = "";
boolean dataFound = false;
- Extract() {
+ Extract() {
}
- void init() throws IngestModuleException {
+ final void init() throws IngestModuleException {
+ try {
+ currentCase = Case.getOpenCase();
+ tskCase = currentCase.getSleuthkitCase();
+ } catch (NoCurrentCaseException ex) {
+ throw new IngestModuleException(Bundle.Extract_indexError_message(), ex);
+ }
+ configExtractor();
+ }
+
+ /**
+ * Override to add any module-specific configuration
+ *
+ * @throws IngestModuleException
+ */
+ void configExtractor() throws IngestModuleException {
}
abstract void process(Content dataSource, IngestJobContext context);
diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java
index 4ef892ffc9..e5d93a91d4 100644
--- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java
+++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java
@@ -393,7 +393,7 @@ class SearchEngineURLQueryAnalyzer extends Extract {
}
@Override
- void init() throws IngestModuleException {
+ void configExtractor() throws IngestModuleException {
try {
PlatformUtil.extractResourceToUserConfigDir(SearchEngineURLQueryAnalyzer.class, XMLFILE, true);
} catch (IOException e) {
diff --git a/TSKVersion.xml b/TSKVersion.xml
new file mode 100644
index 0000000000..544d73ea7f
--- /dev/null
+++ b/TSKVersion.xml
@@ -0,0 +1,3 @@
+
+
+
diff --git a/build.xml b/build.xml
index 0c45446ab2..9a8b8bd507 100644
--- a/build.xml
+++ b/build.xml
@@ -6,7 +6,7 @@
Builds the module suite Autopsy 4.
-
+
@@ -79,6 +79,10 @@
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -96,6 +113,7 @@
+
@@ -279,10 +297,6 @@
-
-
-
-
diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java
index 77345eb853..7aff9be4f3 100644
--- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java
+++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java
@@ -56,6 +56,7 @@ import org.apache.james.mime4j.stream.MimeConfig;
import org.apache.tika.parser.txt.CharsetDetector;
import org.apache.tika.parser.txt.CharsetMatch;
import org.openide.util.NbBundle;
+import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.EncodedFileOutputStream;
@@ -267,8 +268,18 @@ class MboxParser {
* @param email
* @param e
*/
+ @NbBundle.Messages ({"MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."})
private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) {
- String outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator;
+ String outputDirPath;
+ String relModuleOutputPath;
+ try {
+ outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator;
+ relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator;
+ } catch (NoCurrentCaseException ex) {
+ addErrorMessage(Bundle.MboxParser_handleAttch_noOpenCase_errMsg());
+ logger.log(Level.SEVERE, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS
+ return;
+ }
String filename = e.getFilename();
// sanitize name. Had an attachment with a Japanese encoded path that
@@ -325,8 +336,7 @@ class MboxParser {
EmailMessage.Attachment attach = new EmailMessage.Attachment();
attach.setName(filename);
- attach.setLocalPath(ThunderbirdMboxFileIngestModule.getRelModuleOutputPath()
- + File.separator + uniqueFilename);
+ attach.setLocalPath(relModuleOutputPath + uniqueFilename);
attach.setSize(new File(outPath).length());
attach.setEncodingType(TskData.EncodingType.XOR1);
email.addAttachment(attach);
diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java
index 541415e82b..a9a659b475 100644
--- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java
+++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java
@@ -33,6 +33,8 @@ import java.util.List;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.openide.util.NbBundle;
+import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
+import org.sleuthkit.autopsy.ingest.IngestModule;
import org.sleuthkit.autopsy.ingest.IngestMonitor;
import org.sleuthkit.autopsy.ingest.IngestServices;
import static org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule.getRelModuleOutputPath;
@@ -204,9 +206,16 @@ class PstParser {
* @param email
* @param msg
*/
+ @NbBundle.Messages({"PstParser.noOpenCase.errMsg=Exception while getting open case."})
private void extractAttachments(EmailMessage email, PSTMessage msg, long fileID) {
int numberOfAttachments = msg.getNumberOfAttachments();
- String outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator;
+ String outputDirPath;
+ try {
+ outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator;
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
+ return;
+ }
for (int x = 0; x < numberOfAttachments; x++) {
String filename = "";
try {
@@ -246,6 +255,9 @@ class PstParser {
NbBundle.getMessage(this.getClass(), "PstParser.extractAttch.errMsg.failedToExtractToDisk",
filename));
logger.log(Level.WARNING, "Failed to extract attachment from pst file.", ex); //NON-NLS
+ } catch (NoCurrentCaseException ex) {
+ addErrorMessage(Bundle.PstParser_noOpenCase_errMsg());
+ logger.log(Level.SEVERE, Bundle.PstParser_noOpenCase_errMsg(), ex); //NON-NLS
}
}
}
diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
index 10dc2d31ce..1e439cb39d 100644
--- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
+++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
@@ -31,6 +31,7 @@ import java.util.regex.Pattern;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
+import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.Blackboard;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.Logger;
@@ -74,15 +75,26 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
}
@Override
+ @Messages ({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."})
public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context;
- fileManager = Case.getCurrentCase().getServices().getFileManager();
+ try {
+ fileManager = Case.getOpenCase().getServices().getFileManager();
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex);
+ throw new IngestModuleException(Bundle.ThunderbirdMboxFileIngestModule_noOpenCase_errMsg(), ex);
+ }
}
@Override
public ProcessResult process(AbstractFile abstractFile) {
- blackboard = Case.getCurrentCase().getServices().getBlackboard();
+ try {
+ blackboard = Case.getOpenCase().getServices().getBlackboard();
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex);
+ return ProcessResult.ERROR;
+ }
// skip known
if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) {
@@ -133,8 +145,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
*/
@Messages({"ThunderbirdMboxFileIngestModule.processPst.indexError.message=Failed to index encryption detected artifact for keyword search."})
private ProcessResult processPst(AbstractFile abstractFile) {
- String fileName = getTempPath() + File.separator + abstractFile.getName()
+ String fileName;
+ try {
+ fileName = getTempPath() + File.separator + abstractFile.getName()
+ "-" + String.valueOf(abstractFile.getId());
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
+ return ProcessResult.ERROR;
+ }
File file = new File(fileName);
long freeSpace = services.getFreeDiskSpace();
@@ -159,8 +177,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
PstParser.ParseResult result = parser.parse(file, abstractFile.getId());
if (result == PstParser.ParseResult.OK) {
- // parse success: Process email and add artifacts
- processEmails(parser.getResults(), abstractFile);
+ try {
+ // parse success: Process email and add artifacts
+ processEmails(parser.getResults(), abstractFile);
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
+ return ProcessResult.ERROR;
+ }
+
} else if (result == PstParser.ParseResult.ENCRYPT) {
// encrypted pst: Add encrypted file artifact
try {
@@ -225,8 +249,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
emailFolder = emailFolder + mboxFileName;
emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS
- String fileName = getTempPath() + File.separator + abstractFile.getName()
+ String fileName;
+ try {
+ fileName = getTempPath() + File.separator + abstractFile.getName()
+ "-" + String.valueOf(abstractFile.getId());
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
+ return ProcessResult.ERROR;
+ }
File file = new File(fileName);
long freeSpace = services.getFreeDiskSpace();
@@ -249,7 +279,12 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
MboxParser parser = new MboxParser(services, emailFolder);
List emails = parser.parse(file, abstractFile.getId());
- processEmails(emails, abstractFile);
+ try {
+ processEmails(emails, abstractFile);
+ } catch (NoCurrentCaseException ex) {
+ logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
+ return ProcessResult.ERROR;
+ }
if (file.delete() == false) {
logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS
@@ -268,10 +303,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
/**
* Get a path to a temporary folder.
*
- * @return
+ * @throws NoCurrentCaseException if there is no open case.
+ * @return the temporary folder
*/
- public static String getTempPath() {
- String tmpDir = Case.getCurrentCase().getTempDirectory() + File.separator
+ public static String getTempPath() throws NoCurrentCaseException {
+ String tmpDir = Case.getOpenCase().getTempDirectory() + File.separator
+ "EmailParser"; //NON-NLS
File dir = new File(tmpDir);
if (dir.exists() == false) {
@@ -280,8 +316,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
return tmpDir;
}
- public static String getModuleOutputPath() {
- String outDir = Case.getCurrentCase().getModuleDirectory() + File.separator
+ /**
+ * Get a module output folder.
+ *
+ * @throws NoCurrentCaseException if there is no open case.
+ * @return the module output folder
+ */
+ public static String getModuleOutputPath() throws NoCurrentCaseException {
+ String outDir = Case.getOpenCase().getModuleDirectory() + File.separator
+ EmailParserModuleFactory.getModuleName();
File dir = new File(outDir);
if (dir.exists() == false) {
@@ -290,8 +332,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
return outDir;
}
- public static String getRelModuleOutputPath() {
- return Case.getCurrentCase().getModuleOutputDirectoryRelativePath() + File.separator
+ /**
+ * Get a relative path of a module output folder.
+ *
+ * @throws NoCurrentCaseException if there is no open case.
+ * @return the relative path of the module output folder
+ */
+ public static String getRelModuleOutputPath() throws NoCurrentCaseException {
+ return Case.getOpenCase().getModuleOutputDirectoryRelativePath() + File.separator
+ EmailParserModuleFactory.getModuleName();
}
@@ -301,8 +349,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
*
* @param emails
* @param abstractFile
+ * @throws NoCurrentCaseException if there is no open case.
*/
- private void processEmails(List emails, AbstractFile abstractFile) {
+ private void processEmails(List emails, AbstractFile abstractFile) throws NoCurrentCaseException {
List derivedFiles = new ArrayList<>();
@@ -386,9 +435,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
*
* @param email
* @param abstractFile
+ * @throws NoCurrentCaseException if there is no open case.
*/
@Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."})
- private BlackboardArtifact addArtifact(EmailMessage email, AbstractFile abstractFile) {
+ private BlackboardArtifact addArtifact(EmailMessage email, AbstractFile abstractFile) throws NoCurrentCaseException {
BlackboardArtifact bbart = null;
List bbattributes = new ArrayList<>();
String to = email.getRecipients();
@@ -408,11 +458,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
String senderAddress;
senderAddressList.addAll(findEmailAddresess(from));
- AccountFileInstance senderAccountInstance = null;
+ AccountFileInstance senderAccountInstance = null;
+
+ Case openCase = Case.getOpenCase();
+
if (senderAddressList.size() == 1) {
senderAddress = senderAddressList.get(0);
try {
- senderAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile);
+ senderAccountInstance = openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile);
}
catch(TskCoreException ex) {
logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS
@@ -431,7 +484,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
recipientAddresses.forEach((addr) -> {
try {
AccountFileInstance recipientAccountInstance =
- Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr,
+ openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr,
EmailParserModuleFactory.getModuleName(), abstractFile);
recipientAccountInstances.add(recipientAccountInstance);
}
@@ -467,7 +520,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
bbart.addAttributes(bbattributes);
// Add account relationships
- Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL);
+ openCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL);
try {
// index the artifact for keyword search
diff --git a/unix_setup.sh b/unix_setup.sh
new file mode 100755
index 0000000000..93a98ae1bc
--- /dev/null
+++ b/unix_setup.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Verifies programs are installed and copies native code into the Autopsy folder structure
+
+TSK_VERSION=4.6.0
+
+# Verify PhotoRec was installed
+photorec_filepath=/usr/bin/photorec
+if [ -f "$photorec_filepath" ]; then
+ echo "$photorec_filepath found"
+else
+ echo "ERROR: Photorec not found, please install the testdisk package"
+ exit 1
+fi
+
+# Verify Java was installed and configured
+if [ -n "$JAVA_HOME" ]; then
+ if [ -x "$JAVA_HOME/bin/java" ]; then
+ echo "Java found in $JAVA_HOME"
+ else
+ echo "ERROR: Java was not found in $JAVA_HOME"
+ exit 1
+ fi
+else
+ echo "ERROR: JAVA_HOME environment variable must be defined"
+ exit 1
+fi
+
+# Verify Sleuth Kit Java was installed
+sleuthkit_jar_filepath=/usr/share/java/sleuthkit-$TSK_VERSION.jar;
+ext_jar_filepath=$PWD/autopsy/modules/ext/sleuthkit-postgresql-$TSK_VERSION.jar;
+if [ -f "$sleuthkit_jar_filepath" ]; then
+ echo "$sleuthkit_jar_filepath found"
+ echo "Copying into the Autopsy directory"
+ rm $ext_jar_filepath;
+ if [ "$?" -gt 0 ]; then #checking if remove operation failed
+ echo "exiting .."
+ exit 1
+ else
+ cp $sleuthkit_jar_filepath $ext_jar_filepath
+ if [ "$?" -ne 0 ]; then # checking copy operation was successful
+ echo "exiting..."
+ exit 1
+ fi
+ fi
+else
+ echo "ERROR: $sleuthkit_jar_filepath not found, please install the sleuthkit-java.deb file"
+ exit 1
+fi
+
+# make sure it is executable
+chmod +x bin/autopsy
+
+echo "Autopsy is now configured. You can execute bin/autopsy to start it"