From 2669e2cc60528c20797440a8a5b3998bdcd25682 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 26 Feb 2018 11:49:26 -0500 Subject: [PATCH 01/50] Initial changes. --- .../ingestmodule/Bundle.properties | 2 + .../ingestmodule/IngestModule.java | 46 ++++---- .../ingestmodule/IngestModuleFactory.java | 28 ++++- .../ingestmodule/IngestSettings.java | 71 ++++++++++++ .../ingestmodule/IngestSettingsPanel.form | 70 ++++++++++++ .../ingestmodule/IngestSettingsPanel.java | 101 ++++++++++++++++++ 6 files changed, 297 insertions(+), 21 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties new file mode 100755 index 0000000000..c903c40421 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties @@ -0,0 +1,2 @@ +IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings +IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text=Ignore previously seen notable items. diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 09f3c63449..b3bf0505e6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -55,9 +55,11 @@ import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListen */ @Messages({"IngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", "IngestModule.prevCaseComment.text=Previous Case: "}) -class IngestModule implements FileIngestModule { +final class IngestModule implements FileIngestModule { - private final static Logger LOGGER = Logger.getLogger(IngestModule.class.getName()); + static final boolean DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS = false; + + private final static Logger logger = Logger.getLogger(IngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter(); @@ -66,6 +68,12 @@ class IngestModule implements FileIngestModule { private CorrelationDataSource eamDataSource; private Blackboard blackboard; private CorrelationAttribute.Type filesType; + + private final boolean ignorePreviousNotableItems; + + IngestModule(IngestSettings settings) { + ignorePreviousNotableItems = settings.isIgnorePreviousNotableItems(); + } @Override public ProcessResult process(AbstractFile af) { @@ -89,7 +97,7 @@ class IngestModule implements FileIngestModule { try { dbManager = EamDb.getInstance(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); return ProcessResult.ERROR; } @@ -113,7 +121,7 @@ class IngestModule implements FileIngestModule { postCorrelatedBadFileToBlackboard(af, caseDisplayNames); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS return ProcessResult.ERROR; } } @@ -131,7 +139,7 @@ class IngestModule implements FileIngestModule { eamArtifact.addInstance(cefi); dbManager.prepareBulkArtifact(eamArtifact); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS return ProcessResult.ERROR; } @@ -148,19 +156,19 @@ class IngestModule implements FileIngestModule { try { dbManager = EamDb.getInstance(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); return; } try { dbManager.bulkInsertArtifacts(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS } try { Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamCase.getCaseUUID(), eamDataSource.getDeviceID()); - LOGGER.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS + logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS } // TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk. @@ -193,7 +201,7 @@ class IngestModule implements FileIngestModule { // Don't allow sqlite central repo databases to be used for multi user cases if ((Case.getCurrentCase().getCaseType() == Case.CaseType.MULTI_USER_CASE) && (EamDbPlatformEnum.getSelectedPlatform() == EamDbPlatformEnum.SQLITE)) { - LOGGER.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository."); + logger.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository."); throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS } jobId = context.getJobId(); @@ -202,14 +210,14 @@ class IngestModule implements FileIngestModule { try { centralRepoDb = EamDb.getInstance(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS } try { filesType = centralRepoDb.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS } Case autopsyCase = Case.getCurrentCase(); @@ -223,7 +231,7 @@ class IngestModule implements FileIngestModule { try { eamCase = centralRepoDb.newCase(autopsyCase); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error creating new case in ingest module start up.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error creating new case in ingest module start up.", ex); // NON-NLS throw new IngestModuleException("Error creating new case in ingest module start up.", ex); // NON-NLS } } @@ -231,7 +239,7 @@ class IngestModule implements FileIngestModule { try { eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource()); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS } // TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter @@ -245,7 +253,7 @@ class IngestModule implements FileIngestModule { centralRepoDb.newDataSource(eamDataSource); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS } @@ -268,7 +276,7 @@ class IngestModule implements FileIngestModule { // index the artifact for keyword search blackboard.indexArtifact(tifArtifact); } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS + logger.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS } // send inbox message @@ -277,9 +285,9 @@ class IngestModule implements FileIngestModule { // fire event to notify UI of this new artifact services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)); } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS + logger.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS } catch (IllegalStateException ex) { - LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS + logger.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index ed3d4f0915..a0ec1f4329 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; /** * Factory for Central Repository ingest modules @@ -34,8 +35,13 @@ import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; "IngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) public class IngestModuleFactory extends IngestModuleFactoryAdapter { - private static final String VERSION_NUMBER = "0.8.0"; + private static final String VERSION_NUMBER = "0.9.0"; + /** + * Get the name of the module. + * + * @return The module name. + */ static String getModuleName() { return Bundle.IngestModuleFactory_ingestmodule_name(); } @@ -76,5 +82,23 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } + + @Override + public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { + return new IngestSettings(); + } + + @Override + public boolean hasIngestJobSettingsPanel() { + return true; + } + + @Override + public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { + if (!(settings instanceof IngestSettings)) { + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + } + return new IngestSettingsPanel((IngestSettings) settings); + } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java new file mode 100755 index 0000000000..e69e625b85 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java @@ -0,0 +1,71 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; + +/** + * Ingest job settings for the Correlation Engine module. + */ +final class IngestSettings implements IngestModuleIngestJobSettings { + + private static final long serialVersionUID = 1L; + + private boolean ignorePreviousNotableItems; + + /** + * Instantiate the ingest job settings with default values. + */ + IngestSettings() { + this.ignorePreviousNotableItems = IngestModule.DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS; + } + + /** + * Instantiate the ingest job settings. + * + * @param ignorePreviousNotableItems Ignore previously seen notable items. + */ + IngestSettings(boolean ignorePreviousNotableItems) { + this.ignorePreviousNotableItems = ignorePreviousNotableItems; + } + + @Override + public long getVersionNumber() { + return serialVersionUID; + } + + /** + * Are previously identified notable items ignored? + * + * @return True if ignored; otherwise false. + */ + boolean isIgnorePreviousNotableItems() { + return ignorePreviousNotableItems; + } + + /** + * Consider or ignore previously identified notable items. + * + * @param ignorePreviousNotableItems Are previously identified notable items + * ignored? + */ + void setIgnorePreviousNotableItems(boolean ignorePreviousNotableItems) { + this.ignorePreviousNotableItems = ignorePreviousNotableItems; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form new file mode 100755 index 0000000000..c60abee00d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -0,0 +1,70 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java new file mode 100755 index 0000000000..77e04d1528 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -0,0 +1,101 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; + +/** + * Ingest job settings panel for the Correlation Engine module. + */ +final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { + + /** + * Creates new form IngestModulePanel + */ + public IngestSettingsPanel(IngestSettings settings) { + initComponents(); + customizeComponents(settings); + } + + /** + * Update components with values from the ingest job settings. + * + * @param settings The ingest job settings. + */ + private void customizeComponents(IngestSettings settings) { + ignorePreviousNotableItemsCheckbox.setSelected(settings.isIgnorePreviousNotableItems()); + } + + @Override + public IngestModuleIngestJobSettings getSettings() { + return new IngestSettings(ignorePreviousNotableItemsCheckbox.isSelected()); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + ingestSettingsLabel = new javax.swing.JLabel(); + ignorePreviousNotableItemsCheckbox = new javax.swing.JCheckBox(); + + setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE); + + ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(ignorePreviousNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); + getContentPane().setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(10, 10, 10) + .addComponent(ignorePreviousNotableItemsCheckbox)) + .addComponent(ingestSettingsLabel)) + .addContainerGap(83, Short.MAX_VALUE)) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(ingestSettingsLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(ignorePreviousNotableItemsCheckbox) + .addContainerGap(245, Short.MAX_VALUE)) + ); + + pack(); + }// //GEN-END:initComponents + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JCheckBox ignorePreviousNotableItemsCheckbox; + private javax.swing.JLabel ingestSettingsLabel; + // End of variables declaration//GEN-END:variables + +} From bd1f6346a13f8db591295db8a8603eea3ddf6d6a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 26 Feb 2018 13:13:19 -0500 Subject: [PATCH 02/50] Added flag toggle. --- .../ingestmodule/IngestModule.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index b3bf0505e6..49a89555a0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,10 +23,12 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.services.Blackboard; +import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -48,6 +50,7 @@ import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListener; +import org.sleuthkit.datamodel.ContentTag; /** * Ingest module for inserting entries into the Central Repository database on @@ -76,7 +79,7 @@ final class IngestModule implements FileIngestModule { } @Override - public ProcessResult process(AbstractFile af) { + public ProcessResult process(AbstractFile abstractFile) { if (EamDb.isEnabled() == false) { /* * Not signaling an error for now. This is a workaround for the way @@ -86,10 +89,22 @@ final class IngestModule implements FileIngestModule { */ return ProcessResult.OK; } + + if(ignorePreviousNotableItems) { //DLG: + CorrelationAttribute attribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); //DLG: + //DLG: try { + //DLG: List contentTagsList = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByContent(abstractFile); + //DLG: ContentTag tag = contentTagsList.get(0); + //DLG: tag.getId(); + //DLG: } catch (TskCoreException ex) { + //DLG: Exceptions.printStackTrace(ex); //DLG: + //DLG: return ProcessResult.ERROR; + //DLG: } + } //DLG: blackboard = Case.getCurrentCase().getServices().getBlackboard(); - if (!EamArtifactUtil.isValidCentralRepoFile(af)) { + if (!EamArtifactUtil.isValidCentralRepoFile(abstractFile)) { return ProcessResult.OK; } @@ -107,18 +122,18 @@ final class IngestModule implements FileIngestModule { } // get the hash because we're going to correlate it - String md5 = af.getMd5Hash(); + String md5 = abstractFile.getMd5Hash(); if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) { return ProcessResult.OK; } /* Search the central repo to see if this file was previously * marked as being bad. Create artifact if it was. */ - if (af.getKnown() != TskData.FileKnown.KNOWN) { + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && !ignorePreviousNotableItems) { try { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); if (!caseDisplayNames.isEmpty()) { - postCorrelatedBadFileToBlackboard(af, caseDisplayNames); + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNames); } } catch (EamDbException ex) { logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS @@ -132,7 +147,7 @@ final class IngestModule implements FileIngestModule { CorrelationAttributeInstance cefi = new CorrelationAttributeInstance( eamCase, eamDataSource, - af.getParentPath() + af.getName(), + abstractFile.getParentPath() + abstractFile.getName(), null, TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. ); From a569657a05e09f01a3cd795c2ba5a937d8f6dacb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 26 Feb 2018 13:53:49 -0500 Subject: [PATCH 03/50] Fixed compile issues with panel. --- .../ingestmodule/IngestModuleFactory.java | 4 ++-- .../ingestmodule/IngestSettingsPanel.form | 7 ------- .../ingestmodule/IngestSettingsPanel.java | 8 ++------ 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index a0ec1f4329..e9c9a5a88f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -67,8 +67,8 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { } @Override - public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { - return new IngestModule(); + public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { + return new IngestModule((IngestSettings) settings); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form index c60abee00d..fbcf49c00d 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -1,13 +1,6 @@
- - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index 77e04d1528..e7b559e0e5 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -60,15 +60,13 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { ingestSettingsLabel = new javax.swing.JLabel(); ignorePreviousNotableItemsCheckbox = new javax.swing.JCheckBox(); - setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE); - ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ignorePreviousNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text")); // NOI18N - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); - getContentPane().setLayout(layout); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() @@ -89,8 +87,6 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addComponent(ignorePreviousNotableItemsCheckbox) .addContainerGap(245, Short.MAX_VALUE)) ); - - pack(); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables From a518eb721c879e9e51d1c5efa0340b3e50dcca42 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 27 Feb 2018 15:21:34 -0500 Subject: [PATCH 04/50] Cleanup. --- .../ingestmodule/IngestModule.java | 26 ++++++------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 49a89555a0..041c6b9d85 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -71,9 +71,9 @@ final class IngestModule implements FileIngestModule { private CorrelationDataSource eamDataSource; private Blackboard blackboard; private CorrelationAttribute.Type filesType; - + private final boolean ignorePreviousNotableItems; - + IngestModule(IngestSettings settings) { ignorePreviousNotableItems = settings.isIgnorePreviousNotableItems(); } @@ -89,18 +89,6 @@ final class IngestModule implements FileIngestModule { */ return ProcessResult.OK; } - - if(ignorePreviousNotableItems) { //DLG: - CorrelationAttribute attribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); //DLG: - //DLG: try { - //DLG: List contentTagsList = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByContent(abstractFile); - //DLG: ContentTag tag = contentTagsList.get(0); - //DLG: tag.getId(); - //DLG: } catch (TskCoreException ex) { - //DLG: Exceptions.printStackTrace(ex); //DLG: - //DLG: return ProcessResult.ERROR; - //DLG: } - } //DLG: blackboard = Case.getCurrentCase().getServices().getBlackboard(); @@ -127,8 +115,10 @@ final class IngestModule implements FileIngestModule { return ProcessResult.OK; } - /* Search the central repo to see if this file was previously - * marked as being bad. Create artifact if it was. */ + /* + * Search the central repo to see if this file was previously marked as + * being bad. Create artifact if it was. + */ if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && !ignorePreviousNotableItems) { try { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); @@ -149,7 +139,7 @@ final class IngestModule implements FileIngestModule { eamDataSource, abstractFile.getParentPath() + abstractFile.getName(), null, - TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. + TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. ); eamArtifact.addInstance(cefi); dbManager.prepareBulkArtifact(eamArtifact); @@ -250,7 +240,7 @@ final class IngestModule implements FileIngestModule { throw new IngestModuleException("Error creating new case in ingest module start up.", ex); // NON-NLS } } - + try { eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource()); } catch (EamDbException ex) { From 7553c3418c5a41794195fa09a2b24754a7a29ae9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 27 Feb 2018 16:16:21 -0500 Subject: [PATCH 05/50] Cleanup. --- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 3 --- .../centralrepository/ingestmodule/IngestSettingsPanel.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 041c6b9d85..829f0b006e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,12 +23,10 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; -import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.services.Blackboard; -import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -50,7 +48,6 @@ import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListener; -import org.sleuthkit.datamodel.ContentTag; /** * Ingest module for inserting entries into the Central Repository database on diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index e7b559e0e5..f7afdec94e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -27,7 +27,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { /** - * Creates new form IngestModulePanel + * Creates new form IngestSettingsPanel */ public IngestSettingsPanel(IngestSettings settings) { initComponents(); From 32209514efd13b34421dbd4da836d718df493cc1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 5 Mar 2018 10:13:22 -0500 Subject: [PATCH 06/50] Revised. --- .../eventlisteners/CaseEventListener.java | 4 +- .../eventlisteners/IngestEventsListener.java | 60 +++++++++++++++---- .../ingestmodule/Bundle.properties | 2 +- .../ingestmodule/IngestModule.java | 22 +++++-- .../ingestmodule/IngestSettings.java | 28 ++++----- .../ingestmodule/IngestSettingsPanel.form | 12 ++-- .../ingestmodule/IngestSettingsPanel.java | 16 ++--- 7 files changed, 97 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index b053d9df17..807cfef4bd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,7 +41,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamOrganization; import org.sleuthkit.autopsy.coreutils.ThreadUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -464,6 +463,7 @@ final class CaseEventListener implements PropertyChangeListener { if ((null == event.getOldValue()) && (event.getNewValue() instanceof Case)) { Case curCase = (Case) event.getNewValue(); IngestEventsListener.resetCeModuleInstanceCount(); + IngestEventsListener.resetCorrelationModulesFlaggingNotableCount(); if (!EamDb.isEnabled()) { return; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 0877bc1685..2f1efd3785 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,7 +56,8 @@ public class IngestEventsListener { private static final Logger LOGGER = Logger.getLogger(CorrelationAttribute.class.getName()); final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); - private static int ceModuleInstanceCount = 0; + private static int correlationModuleInstanceCount = 0; + private static int correlationModulesFlaggingNotableCount = 0; private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); @@ -87,21 +88,20 @@ public class IngestEventsListener { } /** - * Enable this IngestEventsListener to add contents to the Correlation - * Engine. - * + * Increase the number of IngestEventsListeners adding contents to the + * Correlation Engine. */ public synchronized static void incrementCorrelationEngineModuleCount() { - ceModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method. + correlationModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method. } /** - * Disable this IngestEventsListener from adding contents to the Correlation - * Engine. + * Decrease the number of IngestEventsListeners adding contents to the + * Correlation Engine. */ public synchronized static void decrementCorrelationEngineModuleCount() { if (getCeModuleInstanceCount() > 0) { //prevent it ingestJobCounter from going negative - ceModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method. + correlationModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method. } } @@ -110,7 +110,7 @@ public class IngestEventsListener { * is being run during injest to 0. */ synchronized static void resetCeModuleInstanceCount() { - ceModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset + correlationModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset } /** @@ -120,7 +120,43 @@ public class IngestEventsListener { * @return boolean True for Correlation Engine enabled, False for disabled */ private synchronized static int getCeModuleInstanceCount() { - return ceModuleInstanceCount; + return correlationModuleInstanceCount; + } + + /** + * Increase the number of IngestEventsListeners adding contents to the + * Correlation Engine with notable item flagging enabled. + */ + public synchronized static void incrementCorrelationModulesFlaggingNotableCount() { + correlationModulesFlaggingNotableCount++; + } + + /** + * Decrease the number of IngestEventsListeners adding contents to the + * Correlation Engine with notable item flagging enabled. + */ + public synchronized static void decrementCorrelationModulesFlaggingNotableCount() { + if (correlationModulesFlaggingNotableCount > 0) { + correlationModulesFlaggingNotableCount--; + } + } + + /** + * Reset the counter which keeps track of if the Correlation Engine Module + * is being run during injest and flagging notable items to 0. + */ + synchronized static void resetCorrelationModulesFlaggingNotableCount() { + correlationModulesFlaggingNotableCount = 0; + } + + /** + * Wether or not the Correlation Engine Module is enabled for any of the + * currently running ingest jobs and flagging notable items. + * + * @return boolean True for Correlation Engine enabled, False for disabled + */ + private synchronized static int getCorrelationModulesFlaggingNotableCount() { + return correlationModulesFlaggingNotableCount; } @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", @@ -219,7 +255,7 @@ public class IngestEventsListener { @Override public void run() { - if (!EamDb.isEnabled()) { + if (!EamDb.isEnabled() || getCorrelationModulesFlaggingNotableCount() == 0) { return; } final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties index c903c40421..a525713f7c 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties @@ -1,2 +1,2 @@ IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings -IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text=Ignore previously seen notable items. +IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 829f0b006e..20fbf86753 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -57,7 +57,7 @@ import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListen "IngestModule.prevCaseComment.text=Previous Case: "}) final class IngestModule implements FileIngestModule { - static final boolean DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS = false; + static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = true; private final static Logger logger = Logger.getLogger(IngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); @@ -69,10 +69,14 @@ final class IngestModule implements FileIngestModule { private Blackboard blackboard; private CorrelationAttribute.Type filesType; - private final boolean ignorePreviousNotableItems; + private final boolean flagTaggedNotableItems; + /** + * //DLG: + * @param settings + */ IngestModule(IngestSettings settings) { - ignorePreviousNotableItems = settings.isIgnorePreviousNotableItems(); + flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); } @Override @@ -116,7 +120,7 @@ final class IngestModule implements FileIngestModule { * Search the central repo to see if this file was previously marked as * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && !ignorePreviousNotableItems) { + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { try { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); if (!caseDisplayNames.isEmpty()) { @@ -151,6 +155,11 @@ final class IngestModule implements FileIngestModule { @Override public void shutDown() { IngestEventsListener.decrementCorrelationEngineModuleCount(); + + if (flagTaggedNotableItems) { + IngestEventsListener.decrementCorrelationModulesFlaggingNotableCount(); + } + if ((EamDb.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) { return; } @@ -185,6 +194,11 @@ final class IngestModule implements FileIngestModule { @Override public void startUp(IngestJobContext context) throws IngestModuleException { IngestEventsListener.incrementCorrelationEngineModuleCount(); + + if (flagTaggedNotableItems) { + IngestEventsListener.incrementCorrelationModulesFlaggingNotableCount(); + } + if (EamDb.isEnabled() == false) { /* * Not throwing the customary exception for now. This is a diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java index e69e625b85..32ab9e9f2d 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java @@ -27,22 +27,22 @@ final class IngestSettings implements IngestModuleIngestJobSettings { private static final long serialVersionUID = 1L; - private boolean ignorePreviousNotableItems; + private boolean flagTaggedNotableItems; /** * Instantiate the ingest job settings with default values. */ IngestSettings() { - this.ignorePreviousNotableItems = IngestModule.DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS; + this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS; } /** * Instantiate the ingest job settings. * - * @param ignorePreviousNotableItems Ignore previously seen notable items. + * @param flagTaggedNotableItems Flag previously tagged notable items. */ - IngestSettings(boolean ignorePreviousNotableItems) { - this.ignorePreviousNotableItems = ignorePreviousNotableItems; + IngestSettings(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; } @Override @@ -51,21 +51,21 @@ final class IngestSettings implements IngestModuleIngestJobSettings { } /** - * Are previously identified notable items ignored? + * Are previously tagged notable items to be flagged? * - * @return True if ignored; otherwise false. + * @return True if flagging; otherwise false. */ - boolean isIgnorePreviousNotableItems() { - return ignorePreviousNotableItems; + boolean isFlagTaggedNotableItems() { + return flagTaggedNotableItems; } /** - * Consider or ignore previously identified notable items. + * Flag or ignore previously identified notable items. * - * @param ignorePreviousNotableItems Are previously identified notable items - * ignored? + * @param ignorePreviousNotableItems Are previously tagged notable items to + * be flagged? */ - void setIgnorePreviousNotableItems(boolean ignorePreviousNotableItems) { - this.ignorePreviousNotableItems = ignorePreviousNotableItems; + void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form index fbcf49c00d..2abe207b5a 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -1,6 +1,6 @@ - + @@ -21,11 +21,11 @@ - + - + @@ -35,7 +35,7 @@ - + @@ -52,10 +52,10 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index f7afdec94e..46538f41c2 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -40,12 +40,12 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { * @param settings The ingest job settings. */ private void customizeComponents(IngestSettings settings) { - ignorePreviousNotableItemsCheckbox.setSelected(settings.isIgnorePreviousNotableItems()); + flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems()); } @Override public IngestModuleIngestJobSettings getSettings() { - return new IngestSettings(ignorePreviousNotableItemsCheckbox.isSelected()); + return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected()); } /** @@ -58,12 +58,12 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { private void initComponents() { ingestSettingsLabel = new javax.swing.JLabel(); - ignorePreviousNotableItemsCheckbox = new javax.swing.JCheckBox(); + flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(ignorePreviousNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -74,9 +74,9 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) - .addComponent(ignorePreviousNotableItemsCheckbox)) + .addComponent(flagTaggedNotableItemsCheckbox)) .addComponent(ingestSettingsLabel)) - .addContainerGap(83, Short.MAX_VALUE)) + .addContainerGap(75, Short.MAX_VALUE)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -84,13 +84,13 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addContainerGap() .addComponent(ingestSettingsLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(ignorePreviousNotableItemsCheckbox) + .addComponent(flagTaggedNotableItemsCheckbox) .addContainerGap(245, Short.MAX_VALUE)) ); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JCheckBox ignorePreviousNotableItemsCheckbox; + private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; private javax.swing.JLabel ingestSettingsLabel; // End of variables declaration//GEN-END:variables From b9255ff9c60e676a54f0a31c1345ee0291ad424c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 11:06:02 -0500 Subject: [PATCH 07/50] Added logic for comparing previous instances. --- .../ingestmodule/IngestModule.java | 33 ++++++++++++++++--- .../ingestmodule/IngestSettingsPanel.form | 2 +- .../ingestmodule/IngestSettingsPanel.java | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 20fbf86753..3396eb8ff2 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,6 +23,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; @@ -120,14 +121,36 @@ final class IngestModule implements FileIngestModule { * Search the central repo to see if this file was previously marked as * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { + + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { + CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); try { - List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); - if (!caseDisplayNames.isEmpty()) { - postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNames); + List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( + contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); + String currentCaseDisplayName = Case.getCurrentCase().getDisplayName(); + boolean taggedOutsideCurrentCase = false; + if (!caseDisplayNamesList.isEmpty()) { + for (String name : caseDisplayNamesList) { + if (!name.equals(currentCaseDisplayName)) { + taggedOutsideCurrentCase = true; + break; + } + } + } + + if(flagTaggedNotableItems || !taggedOutsideCurrentCase) { + try { + caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); + if (!caseDisplayNamesList.isEmpty()) { + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); + } + } catch (EamDbException ex) { + logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS + return ProcessResult.ERROR; + } } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS return ProcessResult.ERROR; } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form index 2abe207b5a..564031cb72 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -25,7 +25,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index 46538f41c2..57d4f0a098 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -76,7 +76,7 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addGap(10, 10, 10) .addComponent(flagTaggedNotableItemsCheckbox)) .addComponent(ingestSettingsLabel)) - .addContainerGap(75, Short.MAX_VALUE)) + .addContainerGap(65, Short.MAX_VALUE)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) From 2260dcdc3f4d2662df5fd9607ac1d0469478bbd8 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Tue, 6 Mar 2018 15:28:13 -0500 Subject: [PATCH 08/50] modified swing components in InjestProfileSelectionPanel.java --- .../IngestProfileSelectionPanel.form | 10 +++++----- .../IngestProfileSelectionPanel.java | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form index b5442660c1..265896a7a1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form +++ b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form @@ -37,10 +37,10 @@ - - + + - + @@ -51,9 +51,9 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java index e16fc98aa0..afc3d25f13 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java @@ -274,18 +274,18 @@ final class IngestProfileSelectionPanel extends JPanel { .addComponent(profileListScrollPane) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(ingestSettingsButton, javax.swing.GroupLayout.PREFERRED_SIZE, 128, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(profileListLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 102, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGap(0, 523, Short.MAX_VALUE))) + .addComponent(ingestSettingsButton) + .addComponent(profileListLabel)) + .addGap(0, 458, Short.MAX_VALUE))) .addContainerGap()) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addContainerGap() - .addComponent(profileListLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 27, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(profileListLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(profileListScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 385, Short.MAX_VALUE) + .addComponent(profileListScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 362, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(ingestSettingsButton) .addGap(18, 18, 18)) From 3d5ac054226f2fc926fc78044115422cf2005141 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 17:33:14 -0500 Subject: [PATCH 09/50] Corrected logic to match story criteria. --- .../eventlisteners/IngestEventsListener.java | 12 ++++--- .../ingestmodule/IngestModule.java | 35 +++++-------------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 2f1efd3785..f4e0fd9fbe 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -255,7 +255,7 @@ public class IngestEventsListener { @Override public void run() { - if (!EamDb.isEnabled() || getCorrelationModulesFlaggingNotableCount() == 0) { + if (!EamDb.isEnabled()) { return; } final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue(); @@ -276,10 +276,12 @@ public class IngestEventsListener { // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); - if (!caseDisplayNames.isEmpty()) { - postCorrelatedBadArtifactToBlackboard(bbArtifact, - caseDisplayNames); + if (getCorrelationModulesFlaggingNotableCount() > 0) { + List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); + if (!caseDisplayNames.isEmpty()) { + postCorrelatedBadArtifactToBlackboard(bbArtifact, + caseDisplayNames); + } } eamArtifacts.add(eamArtifact); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 3396eb8ff2..d1362fce63 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -124,34 +124,17 @@ final class IngestModule implements FileIngestModule { if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); - try { - List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( - contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); - String currentCaseDisplayName = Case.getCurrentCase().getDisplayName(); - boolean taggedOutsideCurrentCase = false; - if (!caseDisplayNamesList.isEmpty()) { - for (String name : caseDisplayNamesList) { - if (!name.equals(currentCaseDisplayName)) { - taggedOutsideCurrentCase = true; - break; - } + if (flagTaggedNotableItems) { + try { + List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( + contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); + if (!caseDisplayNamesList.isEmpty()) { + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); } + } catch (EamDbException ex) { + logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS + return ProcessResult.ERROR; } - - if(flagTaggedNotableItems || !taggedOutsideCurrentCase) { - try { - caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); - if (!caseDisplayNamesList.isEmpty()) { - postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); - } - } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS - return ProcessResult.ERROR; - } - } - } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS - return ProcessResult.ERROR; } } From 7a065600f9e98bd4bdb6b73f0f123774e2552d9a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 17:58:42 -0500 Subject: [PATCH 10/50] Fixed a few typos. --- .../eventlisteners/IngestEventsListener.java | 5 +++-- .../centralrepository/ingestmodule/IngestModule.java | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index f4e0fd9fbe..5815e6b527 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -114,7 +114,7 @@ public class IngestEventsListener { } /** - * Wether or not the Correlation Engine Module is enabled for any of the + * Whether or not the Correlation Engine Module is enabled for any of the * currently running ingest jobs. * * @return boolean True for Correlation Engine enabled, False for disabled @@ -153,7 +153,8 @@ public class IngestEventsListener { * Wether or not the Correlation Engine Module is enabled for any of the * currently running ingest jobs and flagging notable items. * - * @return boolean True for Correlation Engine enabled, False for disabled + * @return boolean True for Correlation Engine flagging enabled, False for + * disabled */ private synchronized static int getCorrelationModulesFlaggingNotableCount() { return correlationModulesFlaggingNotableCount; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index d1362fce63..db802950c3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -73,8 +73,9 @@ final class IngestModule implements FileIngestModule { private final boolean flagTaggedNotableItems; /** - * //DLG: - * @param settings + * Instantiate the Correlation Engine ingest module. + * + * @param settings The ingest settings for the module instance. */ IngestModule(IngestSettings settings) { flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); @@ -123,9 +124,9 @@ final class IngestModule implements FileIngestModule { */ if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { - CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); if (flagTaggedNotableItems) { try { + CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); if (!caseDisplayNamesList.isEmpty()) { From eacd201dcd5f9b5985350934ce6ead76f901a95d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 18:00:10 -0500 Subject: [PATCH 11/50] Fixed typo. --- .../centralrepository/eventlisteners/IngestEventsListener.java | 2 +- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 5815e6b527..9e6d488069 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -150,7 +150,7 @@ public class IngestEventsListener { } /** - * Wether or not the Correlation Engine Module is enabled for any of the + * Whether or not the Correlation Engine Module is enabled for any of the * currently running ingest jobs and flagging notable items. * * @return boolean True for Correlation Engine flagging enabled, False for diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index db802950c3..e0e7c2d7bf 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,7 +23,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; -import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; From 43fcae798d1b3992a70518c2e808bbde731ad03a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 22:25:29 -0500 Subject: [PATCH 12/50] Simplified 'process()' logic. --- .../ingestmodule/IngestModule.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index e0e7c2d7bf..865a5d7086 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -122,19 +122,15 @@ final class IngestModule implements FileIngestModule { * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { - if (flagTaggedNotableItems) { - try { - CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); - List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( - contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); - if (!caseDisplayNamesList.isEmpty()) { - postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); - } - } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS - return ProcessResult.ERROR; + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { + try { + List caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); + if (!caseDisplayNamesList.isEmpty()) { + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); } + } catch (EamDbException ex) { + logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS + return ProcessResult.ERROR; } } From 10ad0a004db2b7048f22e4394a24da1ad96fd687 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 22:28:11 -0500 Subject: [PATCH 13/50] Fixed typo. --- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 865a5d7086..8c4bd236a2 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -129,7 +129,7 @@ final class IngestModule implements FileIngestModule { postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS return ProcessResult.ERROR; } } From 8522b322a4acddd51f1ae3419622242334f0532b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 22:43:55 -0500 Subject: [PATCH 14/50] Removed unnecessary initializations. --- .../eventlisteners/IngestEventsListener.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 9e6d488069..bc404e2606 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -56,8 +56,8 @@ public class IngestEventsListener { private static final Logger LOGGER = Logger.getLogger(CorrelationAttribute.class.getName()); final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); - private static int correlationModuleInstanceCount = 0; - private static int correlationModulesFlaggingNotableCount = 0; + private static int correlationModuleInstanceCount; + private static int correlationModulesFlaggingNotableCount; private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); From d3428e926c9716b266082f57528835d7d19ad879 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Wed, 7 Mar 2018 13:49:44 -0500 Subject: [PATCH 15/50] added preinstall script to zip file --- build.xml | 1 + preinstall.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 preinstall.sh diff --git a/build.xml b/build.xml index ef5ad87980..c43701e11d 100644 --- a/build.xml +++ b/build.xml @@ -87,6 +87,7 @@ + diff --git a/preinstall.sh b/preinstall.sh new file mode 100644 index 0000000000..4c4fd388e3 --- /dev/null +++ b/preinstall.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +photorec_filepath=/usr/bin/photorec; +if [ -f "$photorec_filepath" ]; then + echo "photorec found" +else + echo "Photorec not found, please install testdisk for the photorec carver functionality" + echo "run the command: sudo apt-get install testdisk" +fi + +sleuthkit_jar_filepath=/usr/share/java/sleuthkit-4.6.0.jar; +ext_jar_filepath=./autopsy/modules/ext/sleuthkit-postgresql-4.6.0.jar; +if [ -f "$sleuthkit_jar_filepath" ]; then + echo "sleuthkit jarfile found"; + echo "copying sleuthkit-jar file to the autopsy directory"; + rm ./autopsy/modules/ext/sleuthkit-postgresql-4.6.0.jar + if [ ! -f "$ext_jar_filepath" ]; then + cp $sleuthkit_jar_filepath ./autopsy/modules/ext/sleuthkit-postgresql-4.6.0.jar; + echo "Successfully copied sleuthkit-jar file"; + echo "run autopsy"; + fi +else + echo "Sleuthkit-jar not found, please install the sleuthkit-java.deb file" + echo "run the command: sudo apt install ./sleuthkit-java_4.6.0-1_amd64.deb inside the debian file directory" +fi + + From 2000a48a1bb5f7d19f0b3be8761a023f4771d124 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 8 Mar 2018 11:31:36 -0500 Subject: [PATCH 16/50] Module version changed to match application version. --- .../centralrepository/ingestmodule/IngestModuleFactory.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index e9c9a5a88f..d6fe88a51d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -25,6 +25,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; +import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; /** @@ -35,8 +36,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; "IngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) public class IngestModuleFactory extends IngestModuleFactoryAdapter { - private static final String VERSION_NUMBER = "0.9.0"; - /** * Get the name of the module. * @@ -58,7 +57,7 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public String getModuleVersionNumber() { - return VERSION_NUMBER; + return Version.getVersion(); } @Override From c767aa975cc494be49236596efba4c91eed7a177 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 8 Mar 2018 13:43:33 -0500 Subject: [PATCH 17/50] modified preinstall.sh and added regex to change the version --- Core/build.xml | 2 +- build.xml | 7 ++++++- preinstall.sh | 42 +++++++++++++++++++++++++++--------------- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/Core/build.xml b/Core/build.xml index e75c217d15..655e756c25 100644 --- a/Core/build.xml +++ b/Core/build.xml @@ -19,7 +19,7 @@ - + diff --git a/build.xml b/build.xml index c43701e11d..70ec4cb290 100644 --- a/build.xml +++ b/build.xml @@ -87,7 +87,12 @@ - + + + + + + diff --git a/preinstall.sh b/preinstall.sh index 4c4fd388e3..2c1088fabd 100644 --- a/preinstall.sh +++ b/preinstall.sh @@ -1,27 +1,39 @@ #!/bin/bash -photorec_filepath=/usr/bin/photorec; +photorec_filepath=/usr/bin/photorec if [ -f "$photorec_filepath" ]; then - echo "photorec found" + echo "$photorec_filepath found" else echo "Photorec not found, please install testdisk for the photorec carver functionality" echo "run the command: sudo apt-get install testdisk" + exit 1 fi - -sleuthkit_jar_filepath=/usr/share/java/sleuthkit-4.6.0.jar; -ext_jar_filepath=./autopsy/modules/ext/sleuthkit-postgresql-4.6.0.jar; -if [ -f "$sleuthkit_jar_filepath" ]; then - echo "sleuthkit jarfile found"; - echo "copying sleuthkit-jar file to the autopsy directory"; - rm ./autopsy/modules/ext/sleuthkit-postgresql-4.6.0.jar - if [ ! -f "$ext_jar_filepath" ]; then - cp $sleuthkit_jar_filepath ./autopsy/modules/ext/sleuthkit-postgresql-4.6.0.jar; - echo "Successfully copied sleuthkit-jar file"; - echo "run autopsy"; - fi +VERSION=4.6.0 +sleuthkit_jar_filepath=/usr/share/java/sleuthkit-$VERSION.jar; +ext_jar_filepath=$PWD/autopsy/modules/ext/sleuthkit-postgresql-$VERSION.jar; +if [[ -f "$sleuthkit_jar_filepath" ]] && [[ -f "$ext_jar_filepath" ]]; then + echo "$sleuthkit_jar_filepath found" + echo "copying $sleuthkit_jar_filepath to the autopsy directory" + echo "deleting $ext_jar_filepath" + rm $ext_jar_filepath; + if [ "$?" -gt 0 ]; then #checking if remove operation failed + echo "exiting .." + exit 1 + else + echo "Successfully removed $ext_jar_filepath" + cp $sleuthkit_jar_filepath $ext_jar_filepath + if [ "$?" -eq 0 ]; then # checking copy operation was successful + echo "Successfully copied $sleuthkit_jar_filepath" + else + echo "exiting..." + exit 1 + fi + fi else - echo "Sleuthkit-jar not found, please install the sleuthkit-java.deb file" + echo "$sleuthkit_jar_filepath not found, please install the sleuthkit-java.deb file" echo "run the command: sudo apt install ./sleuthkit-java_4.6.0-1_amd64.deb inside the debian file directory" + exit 1 fi +echo "Autopsy is now configured. You can execute bin/autopsy to start it" From 8215e5f7fab0fd953047db988d1396819ad77a24 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 8 Mar 2018 14:59:29 -0500 Subject: [PATCH 18/50] added tsk_version.xml file to change sleuthkit version --- Core/build.xml | 10 +++++----- TSK_VERSION.xml | 3 +++ build.xml | 4 ++-- preinstall.sh => unix_setup.sh | 9 ++++----- 4 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 TSK_VERSION.xml rename preinstall.sh => unix_setup.sh (80%) diff --git a/Core/build.xml b/Core/build.xml index 655e756c25..9fb2bedd6e 100644 --- a/Core/build.xml +++ b/Core/build.xml @@ -6,7 +6,8 @@ Builds, tests, and runs the project org.sleuthkit.autopsy.core - + + @@ -18,8 +19,7 @@ - - + @@ -54,8 +54,8 @@ - + + + diff --git a/build.xml b/build.xml index 70ec4cb290..396268f50e 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ Builds the module suite Autopsy 4. - + @@ -90,7 +90,7 @@ - + diff --git a/preinstall.sh b/unix_setup.sh similarity index 80% rename from preinstall.sh rename to unix_setup.sh index 2c1088fabd..41c57355de 100644 --- a/preinstall.sh +++ b/unix_setup.sh @@ -8,10 +8,10 @@ else echo "run the command: sudo apt-get install testdisk" exit 1 fi -VERSION=4.6.0 -sleuthkit_jar_filepath=/usr/share/java/sleuthkit-$VERSION.jar; -ext_jar_filepath=$PWD/autopsy/modules/ext/sleuthkit-postgresql-$VERSION.jar; -if [[ -f "$sleuthkit_jar_filepath" ]] && [[ -f "$ext_jar_filepath" ]]; then +TSK_VERSION=4.6.0 +sleuthkit_jar_filepath=/usr/share/java/sleuthkit-$TSK_VERSION.jar; +ext_jar_filepath=$PWD/autopsy/modules/ext/sleuthkit-postgresql-$TSK_VERSION.jar; +if [[ -f "$sleuthkit_jar_filepath" ]]; then echo "$sleuthkit_jar_filepath found" echo "copying $sleuthkit_jar_filepath to the autopsy directory" echo "deleting $ext_jar_filepath" @@ -31,7 +31,6 @@ if [[ -f "$sleuthkit_jar_filepath" ]] && [[ -f "$ext_jar_filepath" ]]; then fi else echo "$sleuthkit_jar_filepath not found, please install the sleuthkit-java.deb file" - echo "run the command: sudo apt install ./sleuthkit-java_4.6.0-1_amd64.deb inside the debian file directory" exit 1 fi From 00b59cb904e58faa0f4a8d7a1e39248828c27c54 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 8 Mar 2018 15:49:16 -0500 Subject: [PATCH 19/50] added regex task to the build-zip target --- Core/build.xml | 5 ++--- TSK_VERSION.xml => TSKVersion.xml | 0 build.xml | 3 ++- unix_setup.sh | 7 +++---- 4 files changed, 7 insertions(+), 8 deletions(-) rename TSK_VERSION.xml => TSKVersion.xml (100%) diff --git a/Core/build.xml b/Core/build.xml index 9fb2bedd6e..085e7b0bc7 100644 --- a/Core/build.xml +++ b/Core/build.xml @@ -6,7 +6,7 @@ Builds, tests, and runs the project org.sleuthkit.autopsy.core - + @@ -18,8 +18,7 @@ - - + diff --git a/TSK_VERSION.xml b/TSKVersion.xml similarity index 100% rename from TSK_VERSION.xml rename to TSKVersion.xml diff --git a/build.xml b/build.xml index 396268f50e..fc3e2b673f 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ Builds the module suite Autopsy 4. - + @@ -90,6 +90,7 @@ + diff --git a/unix_setup.sh b/unix_setup.sh index 41c57355de..15f750f880 100644 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -23,10 +23,10 @@ if [[ -f "$sleuthkit_jar_filepath" ]]; then echo "Successfully removed $ext_jar_filepath" cp $sleuthkit_jar_filepath $ext_jar_filepath if [ "$?" -eq 0 ]; then # checking copy operation was successful - echo "Successfully copied $sleuthkit_jar_filepath" + echo "Successfully copied $sleuthkit_jar_filepath" else - echo "exiting..." - exit 1 + echo "exiting..." + exit 1 fi fi else @@ -35,4 +35,3 @@ else fi echo "Autopsy is now configured. You can execute bin/autopsy to start it" - From 70a3807d05acd25c1b5a25b2c50584c5e4f94b36 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 8 Mar 2018 16:27:48 -0500 Subject: [PATCH 20/50] added condition to check JAVA_HOME --- unix_setup.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/unix_setup.sh b/unix_setup.sh index 15f750f880..68caf20c13 100644 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -8,6 +8,15 @@ else echo "run the command: sudo apt-get install testdisk" exit 1 fi + +if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then + echo "found java executable in $JAVA_HOME" +else + echo "Install java and set JAVA_HOME env variable" + echo "See autopsy linux install instructions for more details" + exit 1 +fi + TSK_VERSION=4.6.0 sleuthkit_jar_filepath=/usr/share/java/sleuthkit-$TSK_VERSION.jar; ext_jar_filepath=$PWD/autopsy/modules/ext/sleuthkit-postgresql-$TSK_VERSION.jar; From fe2f9ca391a4410c2505b98befd05847b6fc61ac Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 8 Mar 2018 17:05:40 -0500 Subject: [PATCH 21/50] modified build.xml to do replace version after copying to the zip folder --- build.xml | 2 +- unix_setup.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 unix_setup.sh diff --git a/build.xml b/build.xml index fc3e2b673f..78504fd8b2 100644 --- a/build.xml +++ b/build.xml @@ -90,8 +90,8 @@ - + diff --git a/unix_setup.sh b/unix_setup.sh old mode 100644 new mode 100755 From 863e3b7a1bfb3e9375de0b01b9a6f07f8ed80a8c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\zhaohui" Date: Fri, 9 Mar 2018 18:11:01 -0500 Subject: [PATCH 22/50] 2229: Use getOpenCase() instead of getCurrentCase() --- .../ExplorerNodeActionVisitor.java | 15 ++++- .../directorytree/ExtractUnallocAction.java | 12 ++-- .../autopsy/recentactivity/Extract.java | 12 +++- .../autopsy/thunderbirdparser/MboxParser.java | 16 ++++- .../autopsy/thunderbirdparser/PstParser.java | 14 ++++- .../ThunderbirdMboxFileIngestModule.java | 59 ++++++++++++++----- 6 files changed, 98 insertions(+), 30 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java index 583c2aa157..c4def667aa 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java @@ -23,12 +23,15 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.AbstractAction; import javax.swing.Action; import org.openide.util.NbBundle; import org.openide.util.Utilities; import org.sleuthkit.autopsy.actions.AddContentTagAction; import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; @@ -71,8 +74,12 @@ public class ExplorerNodeActionVisitor extends ContentVisitor.Default visit(final Image img) { List lst = new ArrayList<>(); //TODO lst.add(new ExtractAction("Extract Image", img)); - lst.add(new ExtractUnallocAction( + try { + lst.add(new ExtractUnallocAction( NbBundle.getMessage(this.getClass(), "ExplorerNodeActionVisitor.action.extUnallocToSingleFiles"), img)); + } catch (NoCurrentCaseException ex) { + Logger.getLogger(ExplorerNodeActionVisitor.class.getName()).log(Level.WARNING, "Exception while getting open case.", ex); //NON-NLS + } return lst; } @@ -84,8 +91,12 @@ public class ExplorerNodeActionVisitor extends ContentVisitor.Default visit(final Volume vol) { List lst = new ArrayList<>(); - lst.add(new ExtractUnallocAction( + try { + lst.add(new ExtractUnallocAction( NbBundle.getMessage(this.getClass(), "ExplorerNodeActionVisitor.action.extUnallocToSingleFile"), vol)); + } catch (NoCurrentCaseException ex) { + Logger.getLogger(ExplorerNodeActionVisitor.class.getName()).log(Level.WARNING, "Exception while getting open case.", ex); //NON-NLS + } return lst; } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java index 1960089075..e288fcb2a2 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java @@ -69,14 +69,14 @@ final class ExtractUnallocAction extends AbstractAction { private long currentImage = 0L; private final boolean isImage; - public ExtractUnallocAction(String title, Volume volume) { + public ExtractUnallocAction(String title, Volume volume) throws NoCurrentCaseException { super(title); isImage = false; OutputFileData outputFileData = new OutputFileData(volume); filesToExtract.add(outputFileData); } - public ExtractUnallocAction(String title, Image image) { + public ExtractUnallocAction(String title, Image image) throws NoCurrentCaseException { super(title); isImage = true; currentImage = image.getId(); @@ -596,14 +596,14 @@ final class ExtractUnallocAction extends AbstractAction { * * @param img Image file to be analyzed */ - OutputFileData(Image img) { + OutputFileData(Image img) throws NoCurrentCaseException { this.layoutFiles = getUnallocFiles(img); Collections.sort(layoutFiles, new SortObjId()); this.volumeId = 0; this.imageId = img.getId(); this.imageName = img.getName(); this.fileName = this.imageName + "-Unalloc-" + this.imageId + "-" + 0 + ".dat"; //NON-NLS - this.fileInstance = new File(Case.getCurrentCase().getExportDirectory() + File.separator + this.fileName); + this.fileInstance = new File(Case.getOpenCase().getExportDirectory() + File.separator + this.fileName); this.sizeInBytes = calcSizeInBytes(); } @@ -612,7 +612,7 @@ final class ExtractUnallocAction extends AbstractAction { * * @param volume Volume file to be analyzed */ - OutputFileData(Volume volume) { + OutputFileData(Volume volume) throws NoCurrentCaseException { try { this.imageName = volume.getDataSource().getName(); this.imageId = volume.getDataSource().getId(); @@ -623,7 +623,7 @@ final class ExtractUnallocAction extends AbstractAction { this.imageId = 0; } this.fileName = this.imageName + "-Unalloc-" + this.imageId + "-" + volumeId + ".dat"; //NON-NLS - this.fileInstance = new File(Case.getCurrentCase().getExportDirectory() + File.separator + this.fileName); + this.fileInstance = new File(Case.getOpenCase().getExportDirectory() + File.separator + this.fileName); this.layoutFiles = getUnallocFiles(volume); Collections.sort(layoutFiles, new SortObjId()); this.sizeInBytes = calcSizeInBytes(); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index 4cb6eaf8e3..403e03c9ed 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -41,17 +41,23 @@ import org.sleuthkit.datamodel.*; abstract class Extract { - protected Case currentCase = Case.getCurrentCase(); - protected SleuthkitCase tskCase = currentCase.getSleuthkitCase(); + protected Case currentCase; + protected SleuthkitCase tskCase; private final Logger logger = Logger.getLogger(this.getClass().getName()); private final ArrayList errorMessages = new ArrayList<>(); String moduleName = ""; boolean dataFound = false; - Extract() { + Extract() { } void init() throws IngestModuleException { + try { + currentCase = Case.getOpenCase(); + tskCase = currentCase.getSleuthkitCase(); + } catch (NoCurrentCaseException ex) { + throw new IngestModuleException("Exception while getting open case.", ex); + } } abstract void process(Content dataSource, IngestJobContext context); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java index 77345eb853..bebd365686 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java @@ -56,6 +56,7 @@ import org.apache.james.mime4j.stream.MimeConfig; import org.apache.tika.parser.txt.CharsetDetector; import org.apache.tika.parser.txt.CharsetMatch; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.EncodedFileOutputStream; @@ -267,8 +268,18 @@ class MboxParser { * @param email * @param e */ + @NbBundle.Messages ({"MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { - String outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; + String outputDirPath; + String relModuleOutputPath; + try { + outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; + relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; + } catch (NoCurrentCaseException ex) { + addErrorMessage(Bundle.MboxParser_handleAttch_noOpenCase_errMsg()); + logger.log(Level.INFO, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS + return; + } String filename = e.getFilename(); // sanitize name. Had an attachment with a Japanese encoded path that @@ -325,8 +336,7 @@ class MboxParser { EmailMessage.Attachment attach = new EmailMessage.Attachment(); attach.setName(filename); - attach.setLocalPath(ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() - + File.separator + uniqueFilename); + attach.setLocalPath(relModuleOutputPath + uniqueFilename); attach.setSize(new File(outPath).length()); attach.setEncodingType(TskData.EncodingType.XOR1); email.addAttachment(attach); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java index 541415e82b..367e913007 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java @@ -33,6 +33,8 @@ import java.util.List; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.ingest.IngestModule; import org.sleuthkit.autopsy.ingest.IngestMonitor; import org.sleuthkit.autopsy.ingest.IngestServices; import static org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule.getRelModuleOutputPath; @@ -206,7 +208,15 @@ class PstParser { */ private void extractAttachments(EmailMessage email, PSTMessage msg, long fileID) { int numberOfAttachments = msg.getNumberOfAttachments(); - String outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; + String outputDirPath; + try { + outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; + } catch (NoCurrentCaseException ex) { + addErrorMessage( + NbBundle.getMessage(this.getClass(), "PstParser.extractAttch.errMsg.failedToExtractToDisk", + filename)); + logger.log(Level.WARNING, "Failed to extract attachment from pst file.", ex); //NON-NLS + } for (int x = 0; x < numberOfAttachments; x++) { String filename = ""; try { @@ -237,7 +247,7 @@ class PstParser { attachment.setSize(attach.getFilesize()); attachment.setEncodingType(TskData.EncodingType.XOR1); email.addAttachment(attachment); - } catch (PSTException | IOException | NullPointerException ex) { + } catch (PSTException | IOException | NullPointerException | NoCurrentCaseException ex) { /** * Swallowing null pointer as it is caused by a problem with * getting input stream (library problem). diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 10dc2d31ce..9215fd3e8b 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -28,9 +28,11 @@ import java.util.Set; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.Blackboard; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; @@ -76,13 +78,22 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { @Override public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; - fileManager = Case.getCurrentCase().getServices().getFileManager(); + try { + fileManager = Case.getOpenCase().getServices().getFileManager(); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); + } } @Override public ProcessResult process(AbstractFile abstractFile) { - blackboard = Case.getCurrentCase().getServices().getBlackboard(); + try { + blackboard = Case.getOpenCase().getServices().getBlackboard(); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); + return ProcessResult.ERROR; + } // skip known if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) { @@ -133,8 +144,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { */ @Messages({"ThunderbirdMboxFileIngestModule.processPst.indexError.message=Failed to index encryption detected artifact for keyword search."}) private ProcessResult processPst(AbstractFile abstractFile) { - String fileName = getTempPath() + File.separator + abstractFile.getName() + String fileName; + try { + fileName = getTempPath() + File.separator + abstractFile.getName() + "-" + String.valueOf(abstractFile.getId()); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return ProcessResult.ERROR; + } File file = new File(fileName); long freeSpace = services.getFreeDiskSpace(); @@ -225,8 +242,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { emailFolder = emailFolder + mboxFileName; emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS - String fileName = getTempPath() + File.separator + abstractFile.getName() + String fileName; + try { + fileName = getTempPath() + File.separator + abstractFile.getName() + "-" + String.valueOf(abstractFile.getId()); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return ProcessResult.ERROR; + } File file = new File(fileName); long freeSpace = services.getFreeDiskSpace(); @@ -270,8 +293,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * * @return */ - public static String getTempPath() { - String tmpDir = Case.getCurrentCase().getTempDirectory() + File.separator + public static String getTempPath() throws NoCurrentCaseException { + String tmpDir = Case.getOpenCase().getTempDirectory() + File.separator + "EmailParser"; //NON-NLS File dir = new File(tmpDir); if (dir.exists() == false) { @@ -280,8 +303,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return tmpDir; } - public static String getModuleOutputPath() { - String outDir = Case.getCurrentCase().getModuleDirectory() + File.separator + public static String getModuleOutputPath() throws NoCurrentCaseException { + String outDir = Case.getOpenCase().getModuleDirectory() + File.separator + EmailParserModuleFactory.getModuleName(); File dir = new File(outDir); if (dir.exists() == false) { @@ -290,8 +313,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return outDir; } - public static String getRelModuleOutputPath() { - return Case.getCurrentCase().getModuleOutputDirectoryRelativePath() + File.separator + public static String getRelModuleOutputPath() throws NoCurrentCaseException { + return Case.getOpenCase().getModuleOutputDirectoryRelativePath() + File.separator + EmailParserModuleFactory.getModuleName(); } @@ -408,11 +431,19 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { String senderAddress; senderAddressList.addAll(findEmailAddresess(from)); - AccountFileInstance senderAccountInstance = null; + AccountFileInstance senderAccountInstance = null; + + Case openCase; + try { + openCase = Case.getOpenCase(); + } catch (NoCurrentCaseException ex) { + logger.log(Level.WARNING, "Exception while getting open case.", ex); //NON-NLS + return null; + } if (senderAddressList.size() == 1) { senderAddress = senderAddressList.get(0); try { - senderAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile); + senderAccountInstance = openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile); } catch(TskCoreException ex) { logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS @@ -431,7 +462,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { recipientAddresses.forEach((addr) -> { try { AccountFileInstance recipientAccountInstance = - Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr, + openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr, EmailParserModuleFactory.getModuleName(), abstractFile); recipientAccountInstances.add(recipientAccountInstance); } @@ -467,7 +498,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bbart.addAttributes(bbattributes); // Add account relationships - Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); + openCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); try { // index the artifact for keyword search From 12acbd4ab498ccdef362ef595203bd113923d971 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Mon, 12 Mar 2018 09:01:49 -0400 Subject: [PATCH 23/50] replaced double brackets in if to supprot sh --- unix_setup.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/unix_setup.sh b/unix_setup.sh index 68caf20c13..37ae23221a 100755 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -9,18 +9,22 @@ else exit 1 fi -if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then - echo "found java executable in $JAVA_HOME" +if [ -n "$JAVA_HOME" ]; then + if [ -x "$JAVA_HOME/bin/java" ]; then + echo "found java executable in $JAVA_HOME" + else + echo "no executable found in $JAVA_HOME" + exit 1 + fi else - echo "Install java and set JAVA_HOME env variable" - echo "See autopsy linux install instructions for more details" + echo "Set JAVA_HOME env variable" exit 1 fi TSK_VERSION=4.6.0 sleuthkit_jar_filepath=/usr/share/java/sleuthkit-$TSK_VERSION.jar; ext_jar_filepath=$PWD/autopsy/modules/ext/sleuthkit-postgresql-$TSK_VERSION.jar; -if [[ -f "$sleuthkit_jar_filepath" ]]; then +if [ -f "$sleuthkit_jar_filepath" ]; then echo "$sleuthkit_jar_filepath found" echo "copying $sleuthkit_jar_filepath to the autopsy directory" echo "deleting $ext_jar_filepath" From 4cec521f8b49b4a927ee0a92fde99648bec0622a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\zhaohui" Date: Mon, 12 Mar 2018 16:13:17 -0400 Subject: [PATCH 24/50] 2229: Part 27: Add comments. --- .../directorytree/ExtractUnallocAction.java | 4 ++++ .../ThunderbirdMboxFileIngestModule.java | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java index e288fcb2a2..c229628e26 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java @@ -595,6 +595,8 @@ final class ExtractUnallocAction extends AbstractAction { * Contingency constructor in event no VolumeSystem exists on an Image. * * @param img Image file to be analyzed + * + * @throws NoCurrentCaseException if there is no open case. */ OutputFileData(Image img) throws NoCurrentCaseException { this.layoutFiles = getUnallocFiles(img); @@ -611,6 +613,8 @@ final class ExtractUnallocAction extends AbstractAction { * Default constructor for extracting info from Volumes. * * @param volume Volume file to be analyzed + * + * @throws NoCurrentCaseException if there is no open case. */ OutputFileData(Volume volume) throws NoCurrentCaseException { try { diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 9215fd3e8b..9e0cf9b962 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -291,7 +291,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Get a path to a temporary folder. * - * @return + * @throws NoCurrentCaseException if there is no open case. + * @return the temporary folder */ public static String getTempPath() throws NoCurrentCaseException { String tmpDir = Case.getOpenCase().getTempDirectory() + File.separator @@ -303,6 +304,12 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return tmpDir; } + /** + * Get a module output folder. + * + * @throws NoCurrentCaseException if there is no open case. + * @return the module output folder + */ public static String getModuleOutputPath() throws NoCurrentCaseException { String outDir = Case.getOpenCase().getModuleDirectory() + File.separator + EmailParserModuleFactory.getModuleName(); @@ -313,6 +320,12 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return outDir; } + /** + * Get a relative path of a module output folder. + * + * @throws NoCurrentCaseException if there is no open case. + * @return the relative path of the module output folder + */ public static String getRelModuleOutputPath() throws NoCurrentCaseException { return Case.getOpenCase().getModuleOutputDirectoryRelativePath() + File.separator + EmailParserModuleFactory.getModuleName(); From 5087904d949f221d7e29233d8b99efae27cd96a1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\zhaohui" Date: Mon, 12 Mar 2018 16:24:53 -0400 Subject: [PATCH 25/50] 2229: Part 27 Remove unused imports and correct the log message. --- .../org/sleuthkit/autopsy/thunderbirdparser/PstParser.java | 6 ++---- .../thunderbirdparser/ThunderbirdMboxFileIngestModule.java | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java index 367e913007..a8889a864c 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java @@ -212,10 +212,8 @@ class PstParser { try { outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; } catch (NoCurrentCaseException ex) { - addErrorMessage( - NbBundle.getMessage(this.getClass(), "PstParser.extractAttch.errMsg.failedToExtractToDisk", - filename)); - logger.log(Level.WARNING, "Failed to extract attachment from pst file.", ex); //NON-NLS + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return; } for (int x = 0; x < numberOfAttachments; x++) { String filename = ""; diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 9e0cf9b962..bafa4e8f0c 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; From baf04f327b7d75e225fd8197875ac589e4d33206 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Mon, 12 Mar 2018 16:58:38 -0400 Subject: [PATCH 26/50] Updated comments. --- unix_setup.sh | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/unix_setup.sh b/unix_setup.sh index 37ae23221a..efc2f46b8d 100755 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -1,49 +1,52 @@ #!/bin/bash +# Verifies programs are installed and copies native code into the Autopsy folder structure + +TSK_VERSION=4.6.0 + +# Verify PhotoRec was installed photorec_filepath=/usr/bin/photorec if [ -f "$photorec_filepath" ]; then echo "$photorec_filepath found" else - echo "Photorec not found, please install testdisk for the photorec carver functionality" - echo "run the command: sudo apt-get install testdisk" + echo "ERROR: Photorec not found, please install the testdisk package" exit 1 fi +# Verify Java was installed and configured if [ -n "$JAVA_HOME" ]; then if [ -x "$JAVA_HOME/bin/java" ]; then - echo "found java executable in $JAVA_HOME" + echo "Java found in $JAVA_HOME" else - echo "no executable found in $JAVA_HOME" + echo "ERROR: Java was not found in $JAVA_HOME" exit 1 fi else - echo "Set JAVA_HOME env variable" + echo "ERROR: JAVA_HOME environment variable must be defined" exit 1 fi -TSK_VERSION=4.6.0 +# Verify Sleuth Kit Java was installed sleuthkit_jar_filepath=/usr/share/java/sleuthkit-$TSK_VERSION.jar; ext_jar_filepath=$PWD/autopsy/modules/ext/sleuthkit-postgresql-$TSK_VERSION.jar; if [ -f "$sleuthkit_jar_filepath" ]; then echo "$sleuthkit_jar_filepath found" - echo "copying $sleuthkit_jar_filepath to the autopsy directory" - echo "deleting $ext_jar_filepath" + echo "Copying into the Autopsy directory" rm $ext_jar_filepath; if [ "$?" -gt 0 ]; then #checking if remove operation failed echo "exiting .." exit 1 else - echo "Successfully removed $ext_jar_filepath" cp $sleuthkit_jar_filepath $ext_jar_filepath if [ "$?" -eq 0 ]; then # checking copy operation was successful - echo "Successfully copied $sleuthkit_jar_filepath" + # echo "Successfully copied $sleuthkit_jar_filepath" else echo "exiting..." exit 1 fi fi else - echo "$sleuthkit_jar_filepath not found, please install the sleuthkit-java.deb file" + echo "ERROR: $sleuthkit_jar_filepath not found, please install the sleuthkit-java.deb file" exit 1 fi From 05bef8b8cdc410803220860d54c73bfb6f42dc7f Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Mon, 12 Mar 2018 17:22:01 -0400 Subject: [PATCH 27/50] Update unix_setup.sh --- unix_setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unix_setup.sh b/unix_setup.sh index efc2f46b8d..a1e24fdf9e 100755 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -50,4 +50,7 @@ else exit 1 fi +# make sure it is executable +chmod +x bin/autopsy + echo "Autopsy is now configured. You can execute bin/autopsy to start it" From 0efb49cb2a87c7fea0b327cab897be1d80679db4 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Mon, 12 Mar 2018 17:59:21 -0400 Subject: [PATCH 28/50] changed the build type to RELEASE --- nbproject/project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbproject/project.properties b/nbproject/project.properties index 781ec2c83b..0cb7b69b5e 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -6,8 +6,8 @@ app.name=${branding.token} ### if left unset, version will default to today's date app.version=4.6.0 ### build.type must be one of: DEVELOPMENT, RELEASE -#build.type=RELEASE -build.type=DEVELOPMENT +build.type=RELEASE +#build.type=DEVELOPMENT project.org.netbeans.progress=org-netbeans-api-progress project.org.sleuthkit.autopsy.experimental=Experimental From b812a7981cbed9d25a9e7528b39d5251eaac17ac Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Tue, 13 Mar 2018 10:28:55 -0400 Subject: [PATCH 29/50] central repository config dialog buttons does not disappear anymore --- .../optionspanel/EamDbSettingsDialog.form | 14 +++++++------- .../optionspanel/EamDbSettingsDialog.java | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form index ea06641799..50979ba938 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form @@ -44,7 +44,7 @@ - + @@ -133,7 +133,7 @@ - + @@ -173,29 +173,29 @@ - + - + - + - + - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java index 9b6c474838..3a8b30ef72 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java @@ -245,7 +245,7 @@ public class EamDbSettingsDialog extends JDialog { .addGroup(pnSQLiteSettingsLayout.createSequentialGroup() .addComponent(cbDatabaseType, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(lbSingleUserSqLite, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(lbSingleUserSqLite, javax.swing.GroupLayout.DEFAULT_SIZE, 467, Short.MAX_VALUE) .addGap(9, 9, 9)) .addGroup(pnSQLiteSettingsLayout.createSequentialGroup() .addComponent(tfDatabasePath) @@ -275,25 +275,25 @@ public class EamDbSettingsDialog extends JDialog { .addComponent(lbDatabaseType, javax.swing.GroupLayout.Alignment.TRAILING)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(lbDatabasePath, javax.swing.GroupLayout.PREFERRED_SIZE, 23, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(lbDatabasePath) .addComponent(tfDatabasePath, javax.swing.GroupLayout.PREFERRED_SIZE, 23, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(bnDatabasePathFileOpen)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(tbDbHostname, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(lbHostName, javax.swing.GroupLayout.PREFERRED_SIZE, 22, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(lbHostName)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(tbDbPort, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(lbPort, javax.swing.GroupLayout.PREFERRED_SIZE, 20, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(lbPort)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(tbDbUsername, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(lbUserName, javax.swing.GroupLayout.PREFERRED_SIZE, 20, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(lbUserName)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(jpDbPassword, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(lbUserPassword, javax.swing.GroupLayout.PREFERRED_SIZE, 20, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(lbUserPassword)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(pnSQLiteSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(lbFullDbPath, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) @@ -319,7 +319,7 @@ public class EamDbSettingsDialog extends JDialog { .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) .addComponent(pnSQLiteSettings, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 11, Short.MAX_VALUE) .addComponent(pnButtons, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addGap(10, 10, 10)) ); From 6397c0e9368393f95e4f120231646fb20b12f555 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 13 Mar 2018 10:31:17 -0400 Subject: [PATCH 30/50] Fixed merge issue regarding LOGGER variable. --- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 2d4fd7c798..6acd46f175 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -96,7 +96,7 @@ final class IngestModule implements FileIngestModule { try { blackboard = Case.getOpenCase().getServices().getBlackboard(); } catch (NoCurrentCaseException ex) { - LOGGER.log(Level.SEVERE, "Exception while getting open case.", ex); + logger.log(Level.SEVERE, "Exception while getting open case.", ex); return ProcessResult.ERROR; } @@ -226,7 +226,7 @@ final class IngestModule implements FileIngestModule { try { autopsyCase = Case.getOpenCase(); } catch (NoCurrentCaseException ex) { - LOGGER.log(Level.SEVERE, "Exception while getting open case.", ex); + logger.log(Level.SEVERE, "Exception while getting open case.", ex); throw new IngestModuleException("Exception while getting open case.", ex); } From 43187f2e1cd039f6b043910479cd79dd66c1376b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 13 Mar 2018 14:05:21 -0400 Subject: [PATCH 31/50] Updated IngestEventsListener to track last CE module setting. --- .../eventlisteners/CaseEventListener.java | 2 +- .../eventlisteners/IngestEventsListener.java | 47 ++++++------------- .../ingestmodule/IngestModule.java | 29 ++++++------ 3 files changed, 30 insertions(+), 48 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 0f65dd8130..da11671a08 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -42,6 +42,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamOrganization; import org.sleuthkit.autopsy.coreutils.ThreadUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -480,7 +481,6 @@ final class CaseEventListener implements PropertyChangeListener { if ((null == event.getOldValue()) && (event.getNewValue() instanceof Case)) { Case curCase = (Case) event.getNewValue(); IngestEventsListener.resetCeModuleInstanceCount(); - IngestEventsListener.resetCorrelationModulesFlaggingNotableCount(); if (!EamDb.isEnabled()) { return; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 10bc4f61c3..87f675daa0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -58,7 +58,7 @@ public class IngestEventsListener { final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); private static int correlationModuleInstanceCount; - private static int correlationModulesFlaggingNotableCount; + private static boolean flagNotableItems; private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); @@ -123,42 +123,23 @@ public class IngestEventsListener { private synchronized static int getCeModuleInstanceCount() { return correlationModuleInstanceCount; } - + /** - * Increase the number of IngestEventsListeners adding contents to the - * Correlation Engine with notable item flagging enabled. + * Are notable items being flagged? + * + * @return True if flagging notable items; otherwise false. */ - public synchronized static void incrementCorrelationModulesFlaggingNotableCount() { - correlationModulesFlaggingNotableCount++; + private synchronized static boolean isFlagNotableItems() { + return flagNotableItems; } - + /** - * Decrease the number of IngestEventsListeners adding contents to the - * Correlation Engine with notable item flagging enabled. + * Configure the listener to flag notable items or not. + * + * @param value True to flag notable items; otherwise false. */ - public synchronized static void decrementCorrelationModulesFlaggingNotableCount() { - if (correlationModulesFlaggingNotableCount > 0) { - correlationModulesFlaggingNotableCount--; - } - } - - /** - * Reset the counter which keeps track of if the Correlation Engine Module - * is being run during injest and flagging notable items to 0. - */ - synchronized static void resetCorrelationModulesFlaggingNotableCount() { - correlationModulesFlaggingNotableCount = 0; - } - - /** - * Whether or not the Correlation Engine Module is enabled for any of the - * currently running ingest jobs and flagging notable items. - * - * @return boolean True for Correlation Engine flagging enabled, False for - * disabled - */ - private synchronized static int getCorrelationModulesFlaggingNotableCount() { - return correlationModulesFlaggingNotableCount; + public synchronized static void setFlagNotableItems(boolean value) { + flagNotableItems = value; } @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", @@ -278,7 +259,7 @@ public class IngestEventsListener { // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - if (getCorrelationModulesFlaggingNotableCount() > 0) { + if (isFlagNotableItems()) { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); if (!caseDisplayNames.isEmpty()) { postCorrelatedBadArtifactToBlackboard(bbArtifact, diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 6acd46f175..252b996e17 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -74,7 +74,7 @@ final class IngestModule implements FileIngestModule { /** * Instantiate the Correlation Engine ingest module. - * + * * @param settings The ingest settings for the module instance. */ IngestModule(IngestSettings settings) { @@ -127,7 +127,6 @@ final class IngestModule implements FileIngestModule { * Search the central repo to see if this file was previously marked as * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { try { List caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); @@ -163,11 +162,7 @@ final class IngestModule implements FileIngestModule { @Override public void shutDown() { IngestEventsListener.decrementCorrelationEngineModuleCount(); - - if (flagTaggedNotableItems) { - IngestEventsListener.decrementCorrelationModulesFlaggingNotableCount(); - } - + if ((EamDb.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) { return; } @@ -202,11 +197,17 @@ final class IngestModule implements FileIngestModule { @Override public void startUp(IngestJobContext context) throws IngestModuleException { IngestEventsListener.incrementCorrelationEngineModuleCount(); - - if (flagTaggedNotableItems) { - IngestEventsListener.incrementCorrelationModulesFlaggingNotableCount(); - } - + + /* + * Tell the IngestEventsListener to flag notable items based on the + * current module's configuration. This is a work around for the lack of + * an artifacts pipeline. Note that this can be changed by another + * module instance. All modules are affected by the value. While not + * ideal, this will be good enough until a better solution can be + * posited. + */ + IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); + if (EamDb.isEnabled() == false) { /* * Not throwing the customary exception for now. This is a @@ -227,9 +228,9 @@ final class IngestModule implements FileIngestModule { autopsyCase = Case.getOpenCase(); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); - throw new IngestModuleException("Exception while getting open case.", ex); + throw new IngestModuleException("Exception while getting open case.", ex); } - + // Don't allow sqlite central repo databases to be used for multi user cases if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) && (EamDbPlatformEnum.getSelectedPlatform() == EamDbPlatformEnum.SQLITE)) { From b01b197556c7075c7de432e74204a459fd1447dd Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 13 Mar 2018 14:36:43 -0400 Subject: [PATCH 32/50] Simplified getIngestJobSettingsPanel(). --- .../centralrepository/ingestmodule/IngestModuleFactory.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index d6fe88a51d..26be4930e1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -94,9 +94,6 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { - if (!(settings instanceof IngestSettings)) { - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); - } return new IngestSettingsPanel((IngestSettings) settings); } From 29dfde8d7b0e49fcc236471f0da3d1d797ddc49c Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 14 Mar 2018 12:42:11 -0400 Subject: [PATCH 33/50] fixed error --- unix_setup.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/unix_setup.sh b/unix_setup.sh index a1e24fdf9e..93a98ae1bc 100755 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -38,9 +38,7 @@ if [ -f "$sleuthkit_jar_filepath" ]; then exit 1 else cp $sleuthkit_jar_filepath $ext_jar_filepath - if [ "$?" -eq 0 ]; then # checking copy operation was successful - # echo "Successfully copied $sleuthkit_jar_filepath" - else + if [ "$?" -ne 0 ]; then # checking copy operation was successful echo "exiting..." exit 1 fi From dd60a72c8a4880b130f6390a6492414df5187ba0 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 14 Mar 2018 12:54:16 -0400 Subject: [PATCH 34/50] First cut at changing AID --- .../autoingest/AutoIngestDashboard.java | 94 ++++++++++++++----- 1 file changed, 72 insertions(+), 22 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index 9324f782a3..d626c28dd3 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -31,6 +31,10 @@ import java.util.logging.Level; import javax.swing.DefaultListSelectionModel; import java.awt.Color; import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import javax.swing.JPanel; import javax.swing.JTable; import javax.swing.SwingWorker; @@ -82,6 +86,11 @@ final class AutoIngestDashboard extends JPanel implements Observer { private final DefaultTableModel runningTableModel; private final DefaultTableModel completedTableModel; private AutoIngestMonitor autoIngestMonitor; + + /** + * Maintain a mapping of each service to it's last status update. + */ + private final ConcurrentHashMap statusByService; /** * Creates a dashboard for monitoring an automated ingest cluster. @@ -105,6 +114,8 @@ final class AutoIngestDashboard extends JPanel implements Observer { * Constructs a panel for monitoring an automated ingest cluster. */ private AutoIngestDashboard() { + this.statusByService = new ConcurrentHashMap<>(); + pendingTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0); runningTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0); @@ -112,6 +123,9 @@ final class AutoIngestDashboard extends JPanel implements Observer { completedTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0); initComponents(); + statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), ServicesMonitor.ServiceStatus.DOWN.toString()); + statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), ServicesMonitor.ServiceStatus.DOWN.toString()); + statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), ServicesMonitor.ServiceStatus.DOWN.toString()); setServicesStatusMessage(); initPendingJobsTable(); initRunningJobsTable(); @@ -122,6 +136,25 @@ final class AutoIngestDashboard extends JPanel implements Observer { */ UIManager.put("PopupMenu.consumeEventOnClose", false); } + + /** + * Update status of the services on the dashboard + */ + private void displayServicesStatus() { + tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message", + statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()), + statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), + statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), + statusByService.get(ServicesMonitor.Service.MESSAGING.toString()))); + String upStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); + if (statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()).compareTo(upStatus) != 0 + || statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()).compareTo(upStatus) != 0 + || statusByService.get(ServicesMonitor.Service.MESSAGING.toString()).compareTo(upStatus) != 0) { + tbServicesStatusMessage.setForeground(Color.RED); + } else { + tbServicesStatusMessage.setForeground(Color.BLACK); + } + } /** * Queries the services monitor and sets the text for the services status @@ -129,15 +162,12 @@ final class AutoIngestDashboard extends JPanel implements Observer { */ private void setServicesStatusMessage() { new SwingWorker() { - String caseDatabaseServerStatus = ServicesMonitor.ServiceStatus.DOWN.toString(); - String keywordSearchServiceStatus = ServicesMonitor.ServiceStatus.DOWN.toString(); - String messagingStatus = ServicesMonitor.ServiceStatus.DOWN.toString(); - + @Override protected Void doInBackground() throws Exception { - caseDatabaseServerStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE); - keywordSearchServiceStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH); - messagingStatus = getServiceStatus(ServicesMonitor.Service.MESSAGING); + statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE)); + statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH)); + statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), getServiceStatus(ServicesMonitor.Service.MESSAGING)); return null; } @@ -152,12 +182,14 @@ final class AutoIngestDashboard extends JPanel implements Observer { String serviceStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Unknown"); try { ServicesMonitor servicesMonitor = ServicesMonitor.getInstance(); - serviceStatus = servicesMonitor.getServiceStatus(service.toString()); + return servicesMonitor.getServiceStatus(service.toString()); + + /*serviceStatus = servicesMonitor.getServiceStatus(service.toString()); if (serviceStatus.compareTo(ServicesMonitor.ServiceStatus.UP.toString()) == 0) { serviceStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); } else { serviceStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down"); - } + }*/ } catch (ServicesMonitor.ServicesMonitorException ex) { LOGGER.log(Level.SEVERE, String.format("Dashboard error getting service status for %s", service), ex); } @@ -166,15 +198,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { @Override protected void done() { - tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message", caseDatabaseServerStatus, keywordSearchServiceStatus, keywordSearchServiceStatus, messagingStatus)); - String upStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); - if (caseDatabaseServerStatus.compareTo(upStatus) != 0 - || keywordSearchServiceStatus.compareTo(upStatus) != 0 - || messagingStatus.compareTo(upStatus) != 0) { - tbServicesStatusMessage.setForeground(Color.RED); - } else { - tbServicesStatusMessage.setForeground(Color.BLACK); - } + displayServicesStatus(); } }.execute(); @@ -413,10 +437,36 @@ final class AutoIngestDashboard extends JPanel implements Observer { * auto ingest job tables. */ private void startUp() throws AutoIngestMonitor.AutoIngestMonitorException { - setServicesStatusMessage(); - ServicesMonitor.getInstance().addSubscriber((PropertyChangeEvent evt) -> { - setServicesStatusMessage(); - }); + + PropertyChangeListener propChangeListener = (PropertyChangeEvent evt) -> { + + String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).getDisplayName(); + String status = evt.getNewValue().toString(); + + // if the status update is for an existing service who's status hasn't changed - do nothing. + if (statusByService.containsKey(serviceDisplayName) && status.equals(statusByService.get(serviceDisplayName))) { + return; + } + + if (status.equals(ServicesMonitor.ServiceStatus.UP.toString())) { + LOGGER.log(Level.INFO, "Connection to {0} is up", serviceDisplayName); //NON-NLS + } else if (status.equals(ServicesMonitor.ServiceStatus.DOWN.toString())) { + LOGGER.log(Level.SEVERE, "Connection to {0} is down", serviceDisplayName); //NON-NLS + } else { + LOGGER.log(Level.INFO, "Status for {0} is {1}", new Object[]{serviceDisplayName, status}); //NON-NLS + } + + statusByService.put(serviceDisplayName, status); + displayServicesStatus(); + }; + + // Subscribe to all multi-user services in order to display their status + Set servicesList = new HashSet<>(); + servicesList.add(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()); + servicesList.add(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()); + servicesList.add(ServicesMonitor.Service.MESSAGING.toString()); + ServicesMonitor.getInstance().addSubscriber(servicesList, propChangeListener); + autoIngestMonitor = new AutoIngestMonitor(); autoIngestMonitor.addObserver(this); autoIngestMonitor.startUp(); From fb68dd0fc19288a5279870ffcf72ef990d10b2e3 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 14 Mar 2018 13:24:46 -0400 Subject: [PATCH 35/50] Bug fixes --- .../autopsy/experimental/autoingest/AutoIngestDashboard.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index d626c28dd3..eb5c85e997 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -146,7 +146,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), statusByService.get(ServicesMonitor.Service.MESSAGING.toString()))); - String upStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); + String upStatus = ServicesMonitor.ServiceStatus.UP.toString(); //NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); if (statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()).compareTo(upStatus) != 0 || statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()).compareTo(upStatus) != 0 || statusByService.get(ServicesMonitor.Service.MESSAGING.toString()).compareTo(upStatus) != 0) { @@ -440,7 +440,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { PropertyChangeListener propChangeListener = (PropertyChangeEvent evt) -> { - String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).getDisplayName(); + String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).toString(); String status = evt.getNewValue().toString(); // if the status update is for an existing service who's status hasn't changed - do nothing. From 5b062aade1679a4d3475027e1ba073f963aa2a97 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\zhaohui" Date: Wed, 14 Mar 2018 13:31:13 -0400 Subject: [PATCH 36/50] 2229: Part 27: Update logging level to severe for NoCurrentCaseException and remove incorrect import Logger --- .../ExplorerNodeActionVisitor.java | 6 ++-- .../autopsy/thunderbirdparser/MboxParser.java | 2 +- .../autopsy/thunderbirdparser/PstParser.java | 6 +++- .../ThunderbirdMboxFileIngestModule.java | 33 ++++++++++++------- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java index c4def667aa..27fb56fd68 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.logging.Level; -import java.util.logging.Logger; import javax.swing.AbstractAction; import javax.swing.Action; import org.openide.util.NbBundle; @@ -44,6 +43,7 @@ import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.LocalDirectory; import org.sleuthkit.datamodel.VirtualDirectory; import org.sleuthkit.datamodel.Volume; +import org.sleuthkit.autopsy.coreutils.Logger; public class ExplorerNodeActionVisitor extends ContentVisitor.Default> { @@ -78,7 +78,7 @@ public class ExplorerNodeActionVisitor extends ContentVisitor.Default emails = parser.parse(file, abstractFile.getId()); - processEmails(emails, abstractFile); + try { + processEmails(emails, abstractFile); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return ProcessResult.ERROR; + } if (file.delete() == false) { logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS @@ -336,8 +348,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * * @param emails * @param abstractFile + * @throws NoCurrentCaseException if there is no open case. */ - private void processEmails(List emails, AbstractFile abstractFile) { + private void processEmails(List emails, AbstractFile abstractFile) throws NoCurrentCaseException { List derivedFiles = new ArrayList<>(); @@ -421,9 +434,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * * @param email * @param abstractFile + * @throws NoCurrentCaseException if there is no open case. */ @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."}) - private BlackboardArtifact addArtifact(EmailMessage email, AbstractFile abstractFile) { + private BlackboardArtifact addArtifact(EmailMessage email, AbstractFile abstractFile) throws NoCurrentCaseException { BlackboardArtifact bbart = null; List bbattributes = new ArrayList<>(); String to = email.getRecipients(); @@ -445,13 +459,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { AccountFileInstance senderAccountInstance = null; - Case openCase; - try { - openCase = Case.getOpenCase(); - } catch (NoCurrentCaseException ex) { - logger.log(Level.WARNING, "Exception while getting open case.", ex); //NON-NLS - return null; - } + Case openCase = Case.getOpenCase(); + if (senderAddressList.size() == 1) { senderAddress = senderAddressList.get(0); try { From b6154cffb3b72e8cd157efa3521c1ec6180919d0 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 14 Mar 2018 13:48:06 -0400 Subject: [PATCH 37/50] More fixes --- .../autoingest/AutoIngestDashboard.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index eb5c85e997..3389a64a03 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -123,9 +123,9 @@ final class AutoIngestDashboard extends JPanel implements Observer { completedTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0); initComponents(); - statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), ServicesMonitor.ServiceStatus.DOWN.toString()); - statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), ServicesMonitor.ServiceStatus.DOWN.toString()); - statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), ServicesMonitor.ServiceStatus.DOWN.toString()); + statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down")); + statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down")); + statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down")); setServicesStatusMessage(); initPendingJobsTable(); initRunningJobsTable(); @@ -146,7 +146,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), statusByService.get(ServicesMonitor.Service.MESSAGING.toString()))); - String upStatus = ServicesMonitor.ServiceStatus.UP.toString(); //NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); + String upStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); if (statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()).compareTo(upStatus) != 0 || statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()).compareTo(upStatus) != 0 || statusByService.get(ServicesMonitor.Service.MESSAGING.toString()).compareTo(upStatus) != 0) { @@ -182,14 +182,12 @@ final class AutoIngestDashboard extends JPanel implements Observer { String serviceStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Unknown"); try { ServicesMonitor servicesMonitor = ServicesMonitor.getInstance(); - return servicesMonitor.getServiceStatus(service.toString()); - - /*serviceStatus = servicesMonitor.getServiceStatus(service.toString()); + serviceStatus = servicesMonitor.getServiceStatus(service.toString()); if (serviceStatus.compareTo(ServicesMonitor.ServiceStatus.UP.toString()) == 0) { serviceStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); } else { serviceStatus = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down"); - }*/ + } } catch (ServicesMonitor.ServicesMonitorException ex) { LOGGER.log(Level.SEVERE, String.format("Dashboard error getting service status for %s", service), ex); } @@ -442,20 +440,22 @@ final class AutoIngestDashboard extends JPanel implements Observer { String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).toString(); String status = evt.getNewValue().toString(); - - // if the status update is for an existing service who's status hasn't changed - do nothing. - if (statusByService.containsKey(serviceDisplayName) && status.equals(statusByService.get(serviceDisplayName))) { - return; - } if (status.equals(ServicesMonitor.ServiceStatus.UP.toString())) { + status = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Up"); LOGGER.log(Level.INFO, "Connection to {0} is up", serviceDisplayName); //NON-NLS } else if (status.equals(ServicesMonitor.ServiceStatus.DOWN.toString())) { + status = NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.tbServicesStatusMessage.Message.Down"); LOGGER.log(Level.SEVERE, "Connection to {0} is down", serviceDisplayName); //NON-NLS } else { LOGGER.log(Level.INFO, "Status for {0} is {1}", new Object[]{serviceDisplayName, status}); //NON-NLS } + // if the status update is for an existing service who's status hasn't changed - do nothing. + if (statusByService.containsKey(serviceDisplayName) && status.equals(statusByService.get(serviceDisplayName))) { + return; + } + statusByService.put(serviceDisplayName, status); displayServicesStatus(); }; From ffe6ac2ee1ddbf7d238ec1eda2c32797a54e4276 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 14 Mar 2018 14:06:40 -0400 Subject: [PATCH 38/50] Converted AutoIngestControlPanel to use cached service statuses --- .../autoingest/AutoIngestControlPanel.java | 86 ++++++++++++++----- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java index ea5b7b5ee3..d097882053 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java @@ -38,7 +38,11 @@ import java.util.logging.Level; import javax.swing.DefaultListSelectionModel; import java.awt.Color; import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; import java.io.File; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Logger; import javax.swing.JOptionPane; import javax.swing.JPanel; @@ -149,6 +153,11 @@ public final class AutoIngestControlPanel extends JPanel implements Observer { private Color pendingTableBackground; private Color pendingTablelForeground; + /** + * Maintain a mapping of each service to it's last status update. + */ + private final ConcurrentHashMap statusByService; + /* * The enum is used in conjunction with the DefaultTableModel class to * provide table models for the JTables used to display a view of the @@ -235,6 +244,8 @@ public final class AutoIngestControlPanel extends JPanel implements Observer { * controlling automated ingest for a single node within the cluster. */ private AutoIngestControlPanel() { + + this.statusByService = new ConcurrentHashMap<>(); //Disable the main window so they can only use the dashboard (if we used setVisible the taskBar icon would go away) WindowManager.getDefault().getMainWindow().setEnabled(false); @@ -248,6 +259,9 @@ public final class AutoIngestControlPanel extends JPanel implements Observer { completedTableModel = new AutoIngestTableModel(JobsTableModelColumns.headers, 0); initComponents(); // Generated code. + statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down")); + statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down")); + statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down")); setServicesStatusMessage(); initPendingJobsTable(); initRunningJobsTable(); @@ -260,6 +274,25 @@ public final class AutoIngestControlPanel extends JPanel implements Observer { UIManager.put("PopupMenu.consumeEventOnClose", false); } + /** + * Update status of the services on the dashboard + */ + private void displayServicesStatus() { + tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message", + statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()), + statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), + statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()), + statusByService.get(ServicesMonitor.Service.MESSAGING.toString()))); + String upStatus = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Up"); + if (statusByService.get(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()).compareTo(upStatus) != 0 + || statusByService.get(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()).compareTo(upStatus) != 0 + || statusByService.get(ServicesMonitor.Service.MESSAGING.toString()).compareTo(upStatus) != 0) { + tbServicesStatusMessage.setForeground(Color.RED); + } else { + tbServicesStatusMessage.setForeground(Color.BLACK); + } + } + /** * Queries the services monitor and sets the text for the services status * text box. @@ -274,15 +307,11 @@ public final class AutoIngestControlPanel extends JPanel implements Observer { private void setServicesStatusMessage() { new SwingWorker() { - String caseDatabaseServerStatus = ServicesMonitor.ServiceStatus.DOWN.toString(); - String keywordSearchServiceStatus = ServicesMonitor.ServiceStatus.DOWN.toString(); - String messagingStatus = ServicesMonitor.ServiceStatus.DOWN.toString(); - @Override protected Void doInBackground() throws Exception { - caseDatabaseServerStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE); - keywordSearchServiceStatus = getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH); - messagingStatus = getServiceStatus(ServicesMonitor.Service.MESSAGING); + statusByService.put(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE)); + statusByService.put(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString(), getServiceStatus(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH)); + statusByService.put(ServicesMonitor.Service.MESSAGING.toString(), getServiceStatus(ServicesMonitor.Service.MESSAGING)); return null; } @@ -311,15 +340,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer { @Override protected void done() { - tbServicesStatusMessage.setText(NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message", caseDatabaseServerStatus, keywordSearchServiceStatus, keywordSearchServiceStatus, messagingStatus)); - String upStatus = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Up"); - if (caseDatabaseServerStatus.compareTo(upStatus) != 0 - || keywordSearchServiceStatus.compareTo(upStatus) != 0 - || messagingStatus.compareTo(upStatus) != 0) { - tbServicesStatusMessage.setForeground(Color.RED); - } else { - tbServicesStatusMessage.setForeground(Color.BLACK); - } + displayServicesStatus(); } }.execute(); @@ -682,12 +703,33 @@ public final class AutoIngestControlPanel extends JPanel implements Observer { return; } - /* - * Subscribe to services monitor events. - */ - ServicesMonitor.getInstance().addSubscriber((PropertyChangeEvent evt) -> { - setServicesStatusMessage(); - }); + PropertyChangeListener propChangeListener = (PropertyChangeEvent evt) -> { + + String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).toString(); + String status = evt.getNewValue().toString(); + + if (status.equals(ServicesMonitor.ServiceStatus.UP.toString())) { + status = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Up"); + } else if (status.equals(ServicesMonitor.ServiceStatus.DOWN.toString())) { + status = NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.tbServicesStatusMessage.Message.Down"); + SYS_LOGGER.log(Level.SEVERE, "Connection to {0} is down", serviceDisplayName); //NON-NLS + } + + // if the status update is for an existing service who's status hasn't changed - do nothing. + if (statusByService.containsKey(serviceDisplayName) && status.equals(statusByService.get(serviceDisplayName))) { + return; + } + + statusByService.put(serviceDisplayName, status); + displayServicesStatus(); + }; + + // Subscribe to all multi-user services in order to display their status + Set servicesList = new HashSet<>(); + servicesList.add(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()); + servicesList.add(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString()); + servicesList.add(ServicesMonitor.Service.MESSAGING.toString()); + ServicesMonitor.getInstance().addSubscriber(servicesList, propChangeListener); /* * Register with the AIM as an observer. From 9522e7930f52c55ea3859f2144ea00ff4612a2b7 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 15 Mar 2018 12:31:55 -0400 Subject: [PATCH 39/50] Added restrictions on when flagging can be disabled. --- .../eventlisteners/IngestEventsListener.java | 4 ++-- .../ingestmodule/IngestModule.java | 4 +++- .../ingestmodule/IngestModuleFactory.java | 13 ++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 87f675daa0..44d45e08bc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -120,7 +120,7 @@ public class IngestEventsListener { * * @return boolean True for Correlation Engine enabled, False for disabled */ - private synchronized static int getCeModuleInstanceCount() { + public synchronized static int getCeModuleInstanceCount() { return correlationModuleInstanceCount; } @@ -129,7 +129,7 @@ public class IngestEventsListener { * * @return True if flagging notable items; otherwise false. */ - private synchronized static boolean isFlagNotableItems() { + public synchronized static boolean isFlagNotableItems() { return flagNotableItems; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 252b996e17..fbe85ccbc5 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -206,7 +206,9 @@ final class IngestModule implements FileIngestModule { * ideal, this will be good enough until a better solution can be * posited. */ - IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); + if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagNotableItems()) { + IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); + } if (EamDb.isEnabled() == false) { /* diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index 26be4930e1..6ef03ae00d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -27,6 +27,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; +import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** * Factory for Central Repository ingest modules @@ -94,7 +95,17 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { - return new IngestSettingsPanel((IngestSettings) settings); + if (settings instanceof IngestSettings) { + return new IngestSettingsPanel((IngestSettings) settings); + } + /* + * Compatibility check for older versions. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new IngestSettingsPanel(new IngestSettings()); + } + + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } } From 6f75e3d953d353d53b52138c27baf55496c28387 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 16 Mar 2018 00:23:35 -0400 Subject: [PATCH 40/50] Store flagging value in DataAddedTask. --- .../eventlisteners/IngestEventsListener.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 44d45e08bc..508798f02b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -192,7 +192,7 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt)); + jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, isFlagNotableItems())); break; } } @@ -230,10 +230,12 @@ public class IngestEventsListener { private final EamDb dbManager; private final PropertyChangeEvent event; + private final boolean flagNotableItemsEnabled; - private DataAddedTask(EamDb db, PropertyChangeEvent evt) { + private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled) { dbManager = db; event = evt; + this.flagNotableItemsEnabled = flagNotableItemsEnabled; } @Override @@ -259,7 +261,7 @@ public class IngestEventsListener { // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - if (isFlagNotableItems()) { + if (flagNotableItemsEnabled) { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); if (!caseDisplayNames.isEmpty()) { postCorrelatedBadArtifactToBlackboard(bbArtifact, From c5994bd4078779555964f8ac452ebca396120f5e Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Fri, 16 Mar 2018 09:43:09 -0400 Subject: [PATCH 41/50] added disable experimental module script to build-zip target --- build.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.xml b/build.xml index 78504fd8b2..ae2035fc1b 100644 --- a/build.xml +++ b/build.xml @@ -79,6 +79,10 @@ + + + + - - - - From 7f503055e13617a72330657600103fc3c1d32d6e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 16 Mar 2018 13:01:24 -0400 Subject: [PATCH 42/50] Added additional commentary for setFlagNotableItems() call. --- .../centralrepository/ingestmodule/IngestModule.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index fbe85ccbc5..991da1ad58 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -205,6 +205,12 @@ final class IngestModule implements FileIngestModule { * module instance. All modules are affected by the value. While not * ideal, this will be good enough until a better solution can be * posited. + * + * Note: Flagging cannot be disabled if any other instances of the + * Correlation Engine module are running. This restriction is to prevent + * missing results in the case where the first module is flagging + * notable items, and the proceeding module (with flagging disabled) + * causes the first to stop flagging. */ if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagNotableItems()) { IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); From 55d5827404f95d755e301440bf57a1ebea0cd67b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\zhaohui" Date: Fri, 16 Mar 2018 17:39:58 -0400 Subject: [PATCH 43/50] 2229: Log the exception in the constructor rather than propagate. --- .../directorytree/ExplorerNodeActionVisitor.java | 9 +++------ .../autopsy/directorytree/ExtractUnallocAction.java | 12 +++++++++--- .../sleuthkit/autopsy/recentactivity/Extract.java | 2 +- .../ThunderbirdMboxFileIngestModule.java | 3 ++- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java index 27fb56fd68..99692ea633 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExplorerNodeActionVisitor.java @@ -91,12 +91,9 @@ public class ExplorerNodeActionVisitor extends ContentVisitor.Default visit(final Volume vol) { List lst = new ArrayList<>(); - try { - lst.add(new ExtractUnallocAction( - NbBundle.getMessage(this.getClass(), "ExplorerNodeActionVisitor.action.extUnallocToSingleFile"), vol)); - } catch (NoCurrentCaseException ex) { - Logger.getLogger(ExplorerNodeActionVisitor.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - } + lst.add(new ExtractUnallocAction( + NbBundle.getMessage(this.getClass(), "ExplorerNodeActionVisitor.action.extUnallocToSingleFile"), vol)); + return lst; } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java index c229628e26..893bd94da4 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java @@ -69,11 +69,17 @@ final class ExtractUnallocAction extends AbstractAction { private long currentImage = 0L; private final boolean isImage; - public ExtractUnallocAction(String title, Volume volume) throws NoCurrentCaseException { + public ExtractUnallocAction(String title, Volume volume){ super(title); isImage = false; - OutputFileData outputFileData = new OutputFileData(volume); - filesToExtract.add(outputFileData); + try { + OutputFileData outputFileData = new OutputFileData(volume); + filesToExtract.add(outputFileData); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); + setEnabled(false); + } + } public ExtractUnallocAction(String title, Image image) throws NoCurrentCaseException { diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index 403e03c9ed..b93092d31e 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -56,7 +56,7 @@ abstract class Extract { currentCase = Case.getOpenCase(); tskCase = currentCase.getSleuthkitCase(); } catch (NoCurrentCaseException ex) { - throw new IngestModuleException("Exception while getting open case.", ex); + throw new IngestModuleException(Bundle.Extract_indexError_message(), ex); } } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index e2acdff4e9..1e439cb39d 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -75,13 +75,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } @Override + @Messages ({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; try { fileManager = Case.getOpenCase().getServices().getFileManager(); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); - throw new IngestModuleException("Exception while getting open case.", ex); + throw new IngestModuleException(Bundle.ThunderbirdMboxFileIngestModule_noOpenCase_errMsg(), ex); } } From 2d6a4e22ef262766573fc17236b5eb4c48c127b7 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Mon, 19 Mar 2018 09:00:55 -0400 Subject: [PATCH 44/50] change build.type to development --- nbproject/project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbproject/project.properties b/nbproject/project.properties index 0cb7b69b5e..781ec2c83b 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -6,8 +6,8 @@ app.name=${branding.token} ### if left unset, version will default to today's date app.version=4.6.0 ### build.type must be one of: DEVELOPMENT, RELEASE -build.type=RELEASE -#build.type=DEVELOPMENT +#build.type=RELEASE +build.type=DEVELOPMENT project.org.netbeans.progress=org-netbeans-api-progress project.org.sleuthkit.autopsy.experimental=Experimental From 5c5cde20c8eeed4d813565c07c57c2c559a48136 Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Mon, 19 Mar 2018 09:09:50 -0400 Subject: [PATCH 45/50] Revert "modified swing components in InjestProfileSelectionPanel.java" This reverts commit 2260dcdc3f4d2662df5fd9607ac1d0469478bbd8. --- .../IngestProfileSelectionPanel.form | 10 +++++----- .../IngestProfileSelectionPanel.java | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form index 265896a7a1..b5442660c1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form +++ b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.form @@ -37,10 +37,10 @@ - - + + - + @@ -51,9 +51,9 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java index afc3d25f13..e16fc98aa0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/IngestProfileSelectionPanel.java @@ -274,18 +274,18 @@ final class IngestProfileSelectionPanel extends JPanel { .addComponent(profileListScrollPane) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(ingestSettingsButton) - .addComponent(profileListLabel)) - .addGap(0, 458, Short.MAX_VALUE))) + .addComponent(ingestSettingsButton, javax.swing.GroupLayout.PREFERRED_SIZE, 128, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(profileListLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 102, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGap(0, 523, Short.MAX_VALUE))) .addContainerGap()) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addContainerGap() - .addComponent(profileListLabel) + .addComponent(profileListLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 27, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(profileListScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 362, Short.MAX_VALUE) + .addComponent(profileListScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 385, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(ingestSettingsButton) .addGap(18, 18, 18)) From 7d2657ff70eb6b5c1064a95c726c7d1c1c2e1cd7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 19 Mar 2018 10:27:19 -0400 Subject: [PATCH 46/50] Fix IngestTasksScheduler, IngestManager concurrency issues --- .../autopsy/ingest/DataSourceIngestJob.java | 8 +- .../autopsy/ingest/IngestManager.java | 38 +- .../autopsy/ingest/IngestTasksScheduler.java | 481 ++++++++++-------- 3 files changed, 289 insertions(+), 238 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index a41acd1c58..ebdfb6146f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -518,7 +518,7 @@ final class DataSourceIngestJob { */ if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { logger.log(Level.INFO, "Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS - DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this, this.files); + DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); } else if (this.hasFirstStageDataSourceIngestPipeline()) { logger.log(Level.INFO, "Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); @@ -827,7 +827,7 @@ final class DataSourceIngestJob { } /** - * Adds more files from the data source for this job to the job, i.e., adds + * Adds more files from the data source for this job to the job, e.g., adds * extracted or carved files. Not currently supported for the second stage * of the job. * @@ -835,9 +835,7 @@ final class DataSourceIngestJob { */ void addFiles(List files) { if (DataSourceIngestJob.Stages.FIRST == this.stage) { - for (AbstractFile file : files) { - DataSourceIngestJob.taskScheduler.scheduleFastTrackedFileIngestTask(this, file); - } + DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this, files); } else { DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 65d0162d6d..218626985a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -121,7 +121,7 @@ public class IngestManager { private final AtomicLong nextIngestManagerTaskId = new AtomicLong(0L); private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-start-ingest-jobs-%d").build()); //NON-NLS; private final Map> startIngestJobFutures = new ConcurrentHashMap<>(); - private final Map ingestJobsById = new ConcurrentHashMap<>(); + private final Map ingestJobsById = new HashMap<>(); private final ExecutorService dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS; private final ExecutorService fileLevelIngestJobTasksExecutor; private final ExecutorService eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS; @@ -399,13 +399,17 @@ public class IngestManager { ingestMonitor.start(); } - ingestJobsById.put(job.getId(), job); + synchronized (ingestJobsById) { + ingestJobsById.put(job.getId(), job); + } errors = job.start(); if (errors.isEmpty()) { this.fireIngestJobStarted(job.getId()); IngestManager.logger.log(Level.INFO, "Ingest job {0} started", job.getId()); //NON-NLS } else { - this.ingestJobsById.remove(job.getId()); + synchronized (ingestJobsById) { + this.ingestJobsById.remove(job.getId()); + } for (IngestModuleError error : errors) { logger.log(Level.SEVERE, String.format("Error starting %s ingest module for job %d", error.getModuleDisplayName(), job.getId()), error.getThrowable()); //NON-NLS } @@ -438,7 +442,9 @@ public class IngestManager { */ void finishIngestJob(IngestJob job) { long jobId = job.getId(); - ingestJobsById.remove(jobId); + synchronized (ingestJobsById) { + ingestJobsById.remove(jobId); + } if (!job.isCancelled()) { IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); //NON-NLS fireIngestJobCompleted(jobId); @@ -455,7 +461,9 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { - return !ingestJobsById.isEmpty(); + synchronized (ingestJobsById) { + return !ingestJobsById.isEmpty(); + } } /** @@ -467,9 +475,11 @@ public class IngestManager { startIngestJobFutures.values().forEach((handle) -> { handle.cancel(true); }); - this.ingestJobsById.values().forEach((job) -> { - job.cancel(reason); - }); + synchronized (ingestJobsById) { + this.ingestJobsById.values().forEach((job) -> { + job.cancel(reason); + }); + } } /** @@ -770,9 +780,11 @@ public class IngestManager { */ List getIngestJobSnapshots() { List snapShots = new ArrayList<>(); - ingestJobsById.values().forEach((job) -> { - snapShots.addAll(job.getDataSourceIngestJobSnapshots()); - }); + synchronized (ingestJobsById) { + ingestJobsById.values().forEach((job) -> { + snapShots.addAll(job.getDataSourceIngestJobSnapshots()); + }); + } return snapShots; } @@ -808,7 +820,9 @@ public class IngestManager { public Void call() { try { if (Thread.currentThread().isInterrupted()) { - ingestJobsById.remove(job.getId()); + synchronized (ingestJobsById) { + ingestJobsById.remove(job.getId()); + } return null; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index b9cee95687..7fea572eed 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -21,12 +21,13 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; -import java.util.HashSet; +import java.util.Deque; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; -import java.util.Set; import java.util.TreeSet; import java.util.concurrent.BlockingDeque; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.Level; @@ -40,64 +41,20 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** - * Creates ingest tasks for ingest jobs, queuing the tasks in priority order for - * execution by the ingest manager's ingest threads. + * Creates ingest tasks for data source ingest jobs, queueing the tasks in + * priority order for execution by the ingest manager's ingest threads. */ final class IngestTasksScheduler { - private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName()); private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); + private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName()); private static IngestTasksScheduler instance; - - /** - * Scheduling of data source ingest tasks is accomplished by putting them in - * a FIFO queue to be consumed by the ingest threads, so the queue is - * wrapped in a "dispenser" that implements the IngestTaskQueue interface - * and is exposed via a getter method. - */ - private final LinkedBlockingQueue pendingDataSourceTasks; - private final DataSourceIngestTaskQueue dataSourceTasksDispenser; - - /** - * Scheduling of file ingest tasks is accomplished by "shuffling" them - * through a sequence of internal queues that allows for the interleaving of - * tasks from different ingest jobs based on priority. These scheduling - * queues are: - * - * 1. Root directory tasks (priority queue) - * - * 2. Directory tasks (FIFO queue) - * - * 3. Pending file tasks (LIFO queue). - * - * The pending file tasks queue is LIFO to handle large numbers of files - * extracted from archive files. At least one image has been processed that - * had a folder full of archive files. The queue grew to have thousands of - * entries, as each successive archive file was expanded, so now extracted - * files get added to the front of the queue so that in such a scenario they - * would be processed before the expansion of the next archive file. - * - * Tasks in the pending file tasks queue are ready to be consumed by the - * ingest threads, so the queue is wrapped in a "dispenser" that implements - * the IngestTaskQueue interface and is exposed via a getter method. - */ - private final TreeSet rootDirectoryTasks; - private final List directoryTasks; - private final BlockingDeque pendingFileTasks; - private final FileIngestTaskQueue fileTasksDispenser; - - /** - * The ingest tasks scheduler allows ingest jobs to query it to see if there - * are any tasks in progress for the job. To make this possible, the ingest - * tasks scheduler needs to keep track not only of the tasks in its queues, - * but also of the tasks that have been handed out for processing by the - * ingest threads. Therefore all ingest tasks are added to this list when - * they are created and are not removed when an ingest thread takes an - * ingest task. Instead, the ingest thread calls back into the scheduler - * when the task is completed, at which time the task will be removed from - * this list. - */ - private final Set tasksInProgress; + private final List activeDataSourceTasks; + private final DataSourceIngestTaskQueue dataSourceTaskQueue; + private final TreeSet rootFileTasks; + private final Deque directoryFileTasks; + private final Deque activeFileTasks; + private final FileIngestTaskQueue fileTaskQueue; /** * Gets the ingest tasks scheduler singleton. @@ -113,52 +70,54 @@ final class IngestTasksScheduler { * Constructs an ingest tasks scheduler. */ private IngestTasksScheduler() { - this.pendingDataSourceTasks = new LinkedBlockingQueue<>(); - this.dataSourceTasksDispenser = new DataSourceIngestTaskQueue(); - this.rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); - this.directoryTasks = new ArrayList<>(); - this.pendingFileTasks = new LinkedBlockingDeque<>(); - this.fileTasksDispenser = new FileIngestTaskQueue(); - this.tasksInProgress = new HashSet<>(); + this.activeDataSourceTasks = new ArrayList<>(); + this.dataSourceTaskQueue = new DataSourceIngestTaskQueue(); + this.rootFileTasks = new TreeSet<>(new RootDirectoryTaskComparator()); + this.directoryFileTasks = new LinkedList<>(); + this.activeFileTasks = new LinkedList<>(); + this.fileTaskQueue = new FileIngestTaskQueue(); } /** - * Gets this ingest task scheduler's implementation of the IngestTaskQueue - * interface for data source ingest tasks. + * Gets the data source level ingest tasks queue. The queue is a blocking + * queue intended for use by data source ingest threads. * - * @return The data source ingest tasks queue. + * @return The queue. */ IngestTaskQueue getDataSourceIngestTaskQueue() { - return this.dataSourceTasksDispenser; + return this.dataSourceTaskQueue; } /** - * Gets this ingest task scheduler's implementation of the IngestTaskQueue - * interface for file ingest tasks. + * Gets the file level ingest tasks queue for file ingest threads. The queue + * is a blocking queue intended for use by file ingest threads. * - * @return The file ingest tasks queue. + * @return The queue. */ IngestTaskQueue getFileIngestTaskQueue() { - return this.fileTasksDispenser; + return this.fileTaskQueue; } /** * Schedules a data source level ingest task and file level ingest tasks for - * an ingest job. Either all of the files in the data source or a given - * subset of the files will be scheduled. + * a data source ingest job. Either all of the files in the data source or a + * given subset of the files will be scheduled. * * @param job The data source ingest job. - * @param files A subset of the files for the data source. + * @param files A subset of the files for the data source, possibly empty. */ - synchronized void scheduleIngestTasks(DataSourceIngestJob job, List files) { + synchronized void scheduleIngestTasks(DataSourceIngestJob job) { if (!job.isCancelled()) { - // Scheduling of both a data source ingest task and file ingest tasks - // for a job must be an atomic operation. Otherwise, the data source - // task might be completed before the file tasks are scheduled, - // resulting in a potential false positive when another thread checks - // whether or not all the tasks for the job are completed. + /* + * Scheduling of both the data source ingest task and the initial + * file ingest tasks for a job must be an atomic operation. + * Otherwise, the data source task might be completed before the + * file tasks are scheduled, resulting in a potential false positive + * when another thread checks whether or not all the tasks for the + * job are completed. + */ this.scheduleDataSourceIngestTask(job); - this.scheduleFileIngestTasks(job, files); + this.scheduleFileIngestTasks(job); } } @@ -170,15 +129,12 @@ final class IngestTasksScheduler { synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) { if (!job.isCancelled()) { DataSourceIngestTask task = new DataSourceIngestTask(job); - this.tasksInProgress.add(task); + this.activeDataSourceTasks.add(task); try { - this.pendingDataSourceTasks.put(task); + this.dataSourceTaskQueue.add(task); } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full - * queue. Discard the task and reset the interrupted flag. - */ - this.tasksInProgress.remove(task); + IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while data source ingest thread blocked on a full queue", ex); + this.activeDataSourceTasks.remove(task); Thread.currentThread().interrupt(); } } @@ -190,21 +146,15 @@ final class IngestTasksScheduler { * be scheduled. * * @param job The data source ingest job. - * @param files A subset of the files for the data source. + * @param files A subset of the files for the data source, possibly empty. */ - synchronized void scheduleFileIngestTasks(DataSourceIngestJob job, List files) { + synchronized void scheduleFileIngestTasks(DataSourceIngestJob job) { if (!job.isCancelled()) { - List candidateFiles = new ArrayList<>(); - if (files.isEmpty()) { - getTopLevelFiles(job.getDataSource(), candidateFiles); - } else { - candidateFiles.addAll(files); - } - for (AbstractFile firstLevelFile : candidateFiles) { - FileIngestTask task = new FileIngestTask(job, firstLevelFile); + List candidateFiles = getTopLevelFiles(job.getDataSource()); + for (AbstractFile file : candidateFiles) { + FileIngestTask task = new FileIngestTask(job, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { - this.tasksInProgress.add(task); - this.rootDirectoryTasks.add(task); + this.rootFileTasks.add(task); } } shuffleFileTaskQueues(); @@ -212,73 +162,113 @@ final class IngestTasksScheduler { } /** - * Schedules a file ingest task for a data source ingest job. The task that - * is created is added directly to the pending file tasks queues, i.e., it - * is "fast tracked." + * Schedules file level ingest tasks for a subset of the files in a data + * source ingest job. * - * @param job The data source ingest job. - * @param file A file. + * @param job The data source ingest job. + * @param files A subset of the files for the data source. */ - synchronized void scheduleFastTrackedFileIngestTask(DataSourceIngestJob job, AbstractFile file) { + synchronized void scheduleFileIngestTasks(DataSourceIngestJob job, Collection files) { if (!job.isCancelled()) { - FileIngestTask task = new FileIngestTask(job, file); - if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { - this.tasksInProgress.add(task); - addToPendingFileTasksQueue(task); + final Deque newTasksForIngestThreads = new LinkedList<>(); + for (AbstractFile file : files) { + /* + * The file will be added directly to the front of the queue for + * the ingest threads. + */ + FileIngestTask task = new FileIngestTask(job, file); + if (shouldEnqueueFileTask(task)) { + this.activeFileTasks.addLast(task); // RJCTODO: CHeck this in other method + newTasksForIngestThreads.addLast(task); + } + + /* + * Add the children of the file, if any, either to the front of + * the queue for the ingest threads, in front of the parent + * directory task, or to the end directory task queue. + */ + try { + for (Content child : file.getChildren()) { + if (child instanceof AbstractFile) { + AbstractFile childFile = (AbstractFile) child; + FileIngestTask childTask = new FileIngestTask(job, childFile); + if (childFile.hasChildren()) { + this.directoryFileTasks.add(childTask); + } else if (shouldEnqueueFileTask(childTask)) { + this.activeFileTasks.addLast(childTask); + newTasksForIngestThreads.addFirst(childTask); + } + } + } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", file.getName(), file.getId()), ex); //NON-NLS + } + } + + /* + * Add the newly active tasks into the queue for the file ingest + * threads, AFTER the higher priority tasks that are already queued. + */ + for (FileIngestTask newTask : newTasksForIngestThreads) { + try { + this.fileTaskQueue.tasks.putLast(newTask); + } catch (InterruptedException ex) { + IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while data source ingest thread blocked on a full queue", ex); // RJCTODO: Should this propagate? Correct message + this.activeFileTasks.remove(newTask); + Thread.currentThread().interrupt(); + break; + } } } } /** - * Allows an ingest thread to notify this ingest task scheduler that a task - * has been completed. + * Allows an ingest thread to notify this ingest task scheduler that a data + * source level task has been completed. * * @param task The completed task. */ - synchronized void notifyTaskCompleted(IngestTask task) { - tasksInProgress.remove(task); + synchronized void notifyTaskCompleted(DataSourceIngestTask task) { + this.activeDataSourceTasks.remove(task); + shuffleFileTaskQueues(); } /** - * Queries the task scheduler to determine whether or not all current ingest - * tasks for an ingest job are completed. + * Allows an ingest thread to notify this ingest task scheduler that a file + * level task has been completed. * - * @param job The job for which the query is to be performed. + * @param task + */ + synchronized void notifyTaskCompleted(FileIngestTask task) { + this.activeFileTasks.remove(task); + shuffleFileTaskQueues(); + } + + /** + * Queries the task scheduler to determine whether or not all of the ingest + * tasks for an ingest job have been completed. + * + * @param job The data source ingest job * * @return True or false. */ synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) { - for (IngestTask task : tasksInProgress) { - if (task.getIngestJob().getId() == job.getId()) { - return false; - } - } - return true; + return !hasTasksForJob(this.activeDataSourceTasks, job) + && !hasTasksForJob(this.rootFileTasks, job) + && !hasTasksForJob(this.directoryFileTasks, job) + && !hasTasksForJob(this.activeFileTasks, job); } /** - * Clears the "upstream" task scheduling queues for an ingest job, but does - * nothing about tasks that have already been shuffled into the concurrently - * accessed blocking queues shared with the ingest threads. Note that tasks - * in the "downstream" queues or already taken by the ingest threads will be - * flushed out when the ingest threads call back with their task completed - * notifications. + * Clears the "upstream" task scheduling queues for a data source ingest + * job, but does nothing about tasks that have already been activated, i.e., + * moved into the queue that is consumed by the file ingest threads. * - * @param job The job for which the tasks are to to canceled. + * @param job The data source ingest job */ synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { - /** - * This code should not flush the blocking queues that are concurrently - * accessed by the ingest threads. This is because the "lock striping" - * and "weakly consistent" iterators of these collections make it so - * that this code could have a different view of the queues than the - * ingest threads. It does clean out the directory level tasks before - * they are exploded into file tasks. - */ - long jobId = job.getId(); - this.removeTasksForJob(this.rootDirectoryTasks, jobId); - this.removeTasksForJob(this.directoryTasks, jobId); - this.shuffleFileTaskQueues(); + this.removeTasksForJob(this.rootFileTasks, job); + this.removeTasksForJob(this.directoryFileTasks, job); } /** @@ -289,7 +279,8 @@ final class IngestTasksScheduler { * @param dataSource The data source. * @param topLevelFiles The top level files are added to this list. */ - private static void getTopLevelFiles(Content dataSource, List topLevelFiles) { + private static List getTopLevelFiles(Content dataSource) { + List topLevelFiles = new ArrayList<>(); Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { // The data source is itself a file to be processed. @@ -317,74 +308,108 @@ final class IngestTasksScheduler { } } } + return topLevelFiles; } /** - * "Shuffles" the file task queues to ensure that there is at least one task - * in the pending file ingest tasks queue, as long as there are still file - * ingest tasks to be performed. + * Intelligently queues file ingest tasks for the ingest manager's file + * ingest threads by "shuffling" them through a sequence of queues that + * allows for the interleaving of tasks from different data source ingest + * jobs based on priority. The sequence of queues is: + * + * 1. The root file tasks priority queue, which contains the tasks for the + * roots of data source content sub trees that are being analyzed for the + * data source ingest jobs. For example, typical root tasks for a disk image + * data source would be the tasks for the contents of the root directories + * of the file systems. This queue is a priority queue that attempts to + * ensure that user directory content is analyzed before general file system + * content. It feeds into the directory tasks queue. + * + * 2. The directory file tasks queue, which contains directory tasks + * discovered in the descent through the content sub trees that are being + * analyzed for the data source ingest jobs. It feeds into the active tasks + * queue. + * + * 3. The active file tasks queue, a queue of the file tasks that are either + * in the tasks queue for the file ingest threads or are in the process of + * being analyzed in a file ingest thread. + * + * 4. The file tasks queue for the ingest manager's file ingest threads. */ synchronized private void shuffleFileTaskQueues() { - // This is synchronized because it is called both by synchronized - // methods of this ingest scheduler and an unsynchronized method of its - // file tasks "dispenser". - while (true) { - // Loop until either the pending file tasks queue is NOT empty - // or the upstream queues that feed into it ARE empty. - if (!this.pendingFileTasks.isEmpty()) { - // There are file tasks ready to be consumed, exit. - return; - } - if (this.directoryTasks.isEmpty()) { - if (this.rootDirectoryTasks.isEmpty()) { - // There are no root directory tasks to move into the - // directory queue, exit. - return; + final Deque newTasksForIngestThreads = new LinkedList<>(); + while (this.activeFileTasks.isEmpty()) { + /* + * If the directory file task queue is empty, move the highest + * priority root file task, if there is one, into it. If both the + * root and the directory file task queuess are empty, there is + * nothing left to do. + */ + if (this.directoryFileTasks.isEmpty()) { + if (!this.rootFileTasks.isEmpty()) { + this.directoryFileTasks.add(this.rootFileTasks.pollFirst()); } else { - // Move the next root directory task into the - // directories queue. Note that the task was already - // added to the tasks in progress list when the task was - // created in scheduleFileIngestTasks(). - this.directoryTasks.add(this.rootDirectoryTasks.pollFirst()); + return; } } - // Try to add the most recently added directory from the - // directory tasks queue to the pending file tasks queue. - FileIngestTask directoryTask = this.directoryTasks.remove(this.directoryTasks.size() - 1); + /* + * Try to move the next task from the directory task queue into the + * active file tasks tracking queue, if it passes the filter for the + * job. + */ + final FileIngestTask directoryTask = this.directoryFileTasks.pollLast(); if (shouldEnqueueFileTask(directoryTask)) { - addToPendingFileTasksQueue(directoryTask); - } else { - this.tasksInProgress.remove(directoryTask); + this.activeFileTasks.addFirst(directoryTask); + newTasksForIngestThreads.addFirst(directoryTask); } - // If the directory contains subdirectories or files, try to - // enqueue tasks for them as well. + /* + * If the file or directory from the next that was just activated + * has children, try to queue tasks for the children. Each child + * will go into the directory task queue if it is a directory, or + * into the active file tasks tracking queue if it passes the filter + * for the job. + */ final AbstractFile directory = directoryTask.getFile(); try { for (Content child : directory.getChildren()) { if (child instanceof AbstractFile) { - AbstractFile file = (AbstractFile) child; - FileIngestTask childTask = new FileIngestTask(directoryTask.getIngestJob(), file); - if (file.hasChildren()) { - // Found a subdirectory, put the task in the - // pending directory tasks queue. Note the - // addition of the task to the tasks in progress - // list. This is necessary because this is the - // first appearance of this task in the queues. - this.tasksInProgress.add(childTask); - this.directoryTasks.add(childTask); + AbstractFile childFile = (AbstractFile) child; + FileIngestTask childTask = new FileIngestTask(directoryTask.getIngestJob(), childFile); + if (childFile.hasChildren()) { + this.directoryFileTasks.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - // Found a file, put the task directly into the - // pending file tasks queue. - this.tasksInProgress.add(childTask); - addToPendingFileTasksQueue(childTask); + this.activeFileTasks.add(directoryTask); + /* + * Queue the child file tasks for the ingest threads + * in front of parent directory tasks. + */ + newTasksForIngestThreads.addFirst(directoryTask); } } } } catch (TskCoreException ex) { - String errorMessage = String.format("An error occurred getting the children of %s", directory.getName()); //NON-NLS - logger.log(Level.SEVERE, errorMessage, ex); + logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", directory.getName(), directory.getId()), ex); //NON-NLS + } + + /* + * Add the newly active tasks into the queue for the file ingest + * threads, AFTER the higher priority tasks that are already queued. + */ + for (FileIngestTask newTask : newTasksForIngestThreads) { + try { + this.fileTaskQueue.tasks.putLast(newTask); + } catch (InterruptedException ex) { + /** + * The current thread was interrupted while blocked on a + * full queue. Discard the task and reset the interrupted + * flag. + */ + IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while data source ingest thread blocked on a full queue", ex); // RJCTODO: Should this propagate? Correct message + this.activeFileTasks.remove(newTask); + Thread.currentThread().interrupt(); + } } } } @@ -463,44 +488,44 @@ final class IngestTasksScheduler { } /** - * Adds a file ingest task to the blocking pending tasks queue. + * Checks whether or not a collection of ingest tasks includes a task for a + * given data source ingest job. * - * @param task The task to add. + * @param tasks The tasks. + * @param job The data source ingest job. + * + * @return True if there are no tasks for the job, false otherwise. */ - synchronized private void addToPendingFileTasksQueue(FileIngestTask task) { - try { - this.pendingFileTasks.putFirst(task); - } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full queue. - * Discard the task and reset the interrupted flag. - */ - this.tasksInProgress.remove(task); - Thread.currentThread().interrupt(); + synchronized private boolean hasTasksForJob(Collection tasks, DataSourceIngestJob job) { + long jobId = job.getId(); + for (IngestTask task : tasks) { + if (task.getIngestJob().getId() == jobId) { + return true; + } } + return false; } /** - * Removes all of the ingest tasks associated with an ingest job from a - * tasks queue. The task is removed from the the tasks in progress list as - * well. + * Removes all of the ingest tasks associated with a data source ingest job + * from a tasks collection. * - * @param taskQueue The queue from which to remove the tasks. - * @param jobId The id of the job for which the tasks are to be removed. + * @param tasks The collection from which to remove the tasks. + * @param job THe data source ingest job. */ - synchronized private void removeTasksForJob(Collection taskQueue, long jobId) { - Iterator iterator = taskQueue.iterator(); + synchronized private void removeTasksForJob(Collection tasks, DataSourceIngestJob job) { + long jobId = job.getId(); + Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); if (task.getIngestJob().getId() == jobId) { - this.tasksInProgress.remove(task); iterator.remove(); } } } /** - * Counts the number of ingest tasks in a task queue for a given job. + * Counts the number of ingest tasks in a tasks collection for a given job. * * @param queue The queue for which to count tasks. * @param jobId The id of the job for which the tasks are to be counted. @@ -511,7 +536,7 @@ final class IngestTasksScheduler { Iterator iterator = queue.iterator(); int count = 0; while (iterator.hasNext()) { - IngestTask task = (IngestTask) iterator.next(); + IngestTask task = iterator.next(); if (task.getIngestJob().getId() == jobId) { count++; } @@ -549,8 +574,15 @@ final class IngestTasksScheduler { } } + /** + * Used to prioritize file ingest tasks in the root tasks queue so that + * user content is processed first. + */ private static class AbstractFilePriority { + private AbstractFilePriority() { + } + enum Priority { LAST, LOW, MEDIUM, HIGH @@ -647,10 +679,17 @@ final class IngestTasksScheduler { */ private final class DataSourceIngestTaskQueue implements IngestTaskQueue { + private final BlockingQueue tasks = new LinkedBlockingQueue<>(); + @Override public IngestTask getNextTask() throws InterruptedException { - return IngestTasksScheduler.this.pendingDataSourceTasks.take(); + return tasks.take(); } + + private void add(DataSourceIngestTask task) throws InterruptedException { + this.tasks.put(task); + } + } /** @@ -659,11 +698,11 @@ final class IngestTasksScheduler { */ private final class FileIngestTaskQueue implements IngestTaskQueue { + private final BlockingDeque tasks = new LinkedBlockingDeque<>(); + @Override public IngestTask getNextTask() throws InterruptedException { - FileIngestTask task = IngestTasksScheduler.this.pendingFileTasks.takeFirst(); - shuffleFileTaskQueues(); - return task; + return tasks.takeFirst(); } } @@ -674,10 +713,10 @@ final class IngestTasksScheduler { class IngestJobTasksSnapshot { private final long jobId; + private final long dsQueueSize; private final long rootQueueSize; private final long dirQueueSize; private final long fileQueueSize; - private final long dsQueueSize; private final long runningListSize; /** @@ -687,11 +726,11 @@ final class IngestTasksScheduler { */ IngestJobTasksSnapshot(long jobId) { this.jobId = jobId; - this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootDirectoryTasks, jobId); - this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.directoryTasks, jobId); - this.fileQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingFileTasks, jobId); - this.dsQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingDataSourceTasks, jobId); - this.runningListSize = countTasksForJob(IngestTasksScheduler.this.tasksInProgress, jobId); + this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootFileTasks, jobId); + this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.directoryFileTasks, jobId); + this.fileQueueSize = countTasksForJob(IngestTasksScheduler.this.fileTaskQueue.tasks, jobId); + this.dsQueueSize = countTasksForJob(IngestTasksScheduler.this.dataSourceTaskQueue.tasks, jobId); + this.runningListSize = countTasksForJob(IngestTasksScheduler.this.activeDataSourceTasks, jobId) + countTasksForJob(IngestTasksScheduler.this.activeFileTasks, jobId); } /** From 8c207a9bf9bf34d6c7840778d45a32ac81bd0f2c Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 19 Mar 2018 13:13:37 -0400 Subject: [PATCH 47/50] Make sure the Extract init() method is always called --- .../org/sleuthkit/autopsy/recentactivity/Extract.java | 11 ++++++++++- .../recentactivity/SearchEngineURLQueryAnalyzer.java | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index b93092d31e..8ae8dc0261 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -51,13 +51,22 @@ abstract class Extract { Extract() { } - void init() throws IngestModuleException { + final void init() throws IngestModuleException { try { currentCase = Case.getOpenCase(); tskCase = currentCase.getSleuthkitCase(); } catch (NoCurrentCaseException ex) { throw new IngestModuleException(Bundle.Extract_indexError_message(), ex); } + configExtractor(); + } + + /** + * Override to add any module-specific configuration + * + * @throws IngestModuleException + */ + void configExtractor() throws IngestModuleException { } abstract void process(Content dataSource, IngestJobContext context); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index 4ef892ffc9..e5d93a91d4 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -393,7 +393,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { } @Override - void init() throws IngestModuleException { + void configExtractor() throws IngestModuleException { try { PlatformUtil.extractResourceToUserConfigDir(SearchEngineURLQueryAnalyzer.class, XMLFILE, true); } catch (IOException e) { From d2536e83d5f56ad05c839edddf6da08261e31b7f Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 19 Mar 2018 13:40:04 -0400 Subject: [PATCH 48/50] Fix doxygen warnings --- .../ingest/runIngestModuleWizard/RunIngestModulesAction.java | 2 +- .../sleuthkit/autopsy/modules/interestingitems/FilesSet.java | 2 +- .../src/org/sleuthkit/autopsy/keywordsearch/Server.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java index d322d1026f..4698b68006 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/runIngestModuleWizard/RunIngestModulesAction.java @@ -93,7 +93,7 @@ public final class RunIngestModulesAction extends AbstractAction { * Constructs an action that invokes the Run Ingest Modules wizard for the * children of a file. * - * @param file The file. + * @param parentFile The file. */ public RunIngestModulesAction(AbstractFile parentFile) { this.putValue(Action.NAME, Bundle.RunIngestModulesAction_name()); diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java index 0dbb0300f2..0f8b009c52 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java @@ -547,7 +547,7 @@ public final class FilesSet implements Serializable { /** * Construct a meta-type condition. * - * @param metaType The meta-type to match, must. + * @param type The meta-type to match, must. */ public MetaTypeCondition(Type type) { this.type = type; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 23380b15ae..416f2b394f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -874,7 +874,7 @@ public class Server { * if this does not exist then no server is recorded. * * Format of solrServerList.txt: - * , + * (host),(port) * Ex: 10.1.2.34,8983 * * @param rootOutputDirectory From 0d4a2315cd67659f48eed00a126b7ba64d2ba9ce Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 19 Mar 2018 19:09:36 -0400 Subject: [PATCH 49/50] Fix IngestTasksScheduler, IngestManager concurrency issues --- .../autopsy/ingest/IngestTasksScheduler.java | 295 +++++++++--------- 1 file changed, 155 insertions(+), 140 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 7fea572eed..2bc1769bc6 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -49,12 +49,12 @@ final class IngestTasksScheduler { private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName()); private static IngestTasksScheduler instance; - private final List activeDataSourceTasks; - private final DataSourceIngestTaskQueue dataSourceTaskQueue; - private final TreeSet rootFileTasks; - private final Deque directoryFileTasks; - private final Deque activeFileTasks; - private final FileIngestTaskQueue fileTaskQueue; + private final DataSourceIngestTaskQueue dataSourceTaskQueueForIngestThreads; + private final List queuedAndRunningDataSourceTasks; + private final TreeSet rootFileTaskQueue; + private final Deque directoryFileTaskQueue; + private final FileIngestTaskQueue fileTaskQueueForIngestThreads; + private final List queuedAndRunningFileTasks; /** * Gets the ingest tasks scheduler singleton. @@ -70,41 +70,40 @@ final class IngestTasksScheduler { * Constructs an ingest tasks scheduler. */ private IngestTasksScheduler() { - this.activeDataSourceTasks = new ArrayList<>(); - this.dataSourceTaskQueue = new DataSourceIngestTaskQueue(); - this.rootFileTasks = new TreeSet<>(new RootDirectoryTaskComparator()); - this.directoryFileTasks = new LinkedList<>(); - this.activeFileTasks = new LinkedList<>(); - this.fileTaskQueue = new FileIngestTaskQueue(); + this.queuedAndRunningDataSourceTasks = new LinkedList<>(); + this.dataSourceTaskQueueForIngestThreads = new DataSourceIngestTaskQueue(); + this.rootFileTaskQueue = new TreeSet<>(new RootDirectoryTaskComparator()); + this.directoryFileTaskQueue = new LinkedList<>(); + this.queuedAndRunningFileTasks = new LinkedList<>(); + this.fileTaskQueueForIngestThreads = new FileIngestTaskQueue(); } /** - * Gets the data source level ingest tasks queue. The queue is a blocking - * queue intended for use by data source ingest threads. + * Gets the data source level ingest tasks queue. This queue is a blocking + * queue intended for use by the ingest manager's data source ingest + * threads. * * @return The queue. */ IngestTaskQueue getDataSourceIngestTaskQueue() { - return this.dataSourceTaskQueue; + return this.dataSourceTaskQueueForIngestThreads; } /** - * Gets the file level ingest tasks queue for file ingest threads. The queue - * is a blocking queue intended for use by file ingest threads. + * Gets the file level ingest tasks queue. This queue is a blocking queue + * intended for use by the ingest manager's file ingest threads. * * @return The queue. */ IngestTaskQueue getFileIngestTaskQueue() { - return this.fileTaskQueue; + return this.fileTaskQueueForIngestThreads; } /** * Schedules a data source level ingest task and file level ingest tasks for - * a data source ingest job. Either all of the files in the data source or a - * given subset of the files will be scheduled. + * a data source ingest job. * - * @param job The data source ingest job. - * @param files A subset of the files for the data source, possibly empty. + * @param job The data source ingest job. */ synchronized void scheduleIngestTasks(DataSourceIngestJob job) { if (!job.isCancelled()) { @@ -129,24 +128,21 @@ final class IngestTasksScheduler { synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) { if (!job.isCancelled()) { DataSourceIngestTask task = new DataSourceIngestTask(job); - this.activeDataSourceTasks.add(task); + this.queuedAndRunningDataSourceTasks.add(task); try { - this.dataSourceTaskQueue.add(task); + this.dataSourceTaskQueueForIngestThreads.add(task); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while data source ingest thread blocked on a full queue", ex); - this.activeDataSourceTasks.remove(task); + IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while a data source ingest thread was blocked on a full queue", ex); + this.queuedAndRunningDataSourceTasks.remove(task); Thread.currentThread().interrupt(); } } } /** - * Schedules file level ingest tasks for a data source ingest job. Either - * all of the files in the data source or a given subset of the files will - * be scheduled. + * Schedules file level ingest tasks for a data source ingest job. * - * @param job The data source ingest job. - * @param files A subset of the files for the data source, possibly empty. + * @param job The data source ingest job. */ synchronized void scheduleFileIngestTasks(DataSourceIngestJob job) { if (!job.isCancelled()) { @@ -154,7 +150,7 @@ final class IngestTasksScheduler { for (AbstractFile file : candidateFiles) { FileIngestTask task = new FileIngestTask(job, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { - this.rootFileTasks.add(task); + this.rootFileTaskQueue.add(task); } } shuffleFileTaskQueues(); @@ -162,7 +158,7 @@ final class IngestTasksScheduler { } /** - * Schedules file level ingest tasks for a subset of the files in a data + * Schedules file level ingest tasks for a subset of the files for a data * source ingest job. * * @param job The data source ingest job. @@ -170,22 +166,27 @@ final class IngestTasksScheduler { */ synchronized void scheduleFileIngestTasks(DataSourceIngestJob job, Collection files) { if (!job.isCancelled()) { - final Deque newTasksForIngestThreads = new LinkedList<>(); + List newTasks = new LinkedList<>(); for (AbstractFile file : files) { /* - * The file will be added directly to the front of the queue for - * the ingest threads. + * Put the file directly into the queue for the file ingest + * threads, if it passes the filter for the job. The file is + * added to the queue for the ingest threads BEFORE the other + * queued tasks because the primary use case for this method is + * adding derived files from a higher priority task that + * preceded the tasks currently in the queue. */ FileIngestTask task = new FileIngestTask(job, file); if (shouldEnqueueFileTask(task)) { - this.activeFileTasks.addLast(task); // RJCTODO: CHeck this in other method - newTasksForIngestThreads.addLast(task); + newTasks.add(task); } /* - * Add the children of the file, if any, either to the front of - * the queue for the ingest threads, in front of the parent - * directory task, or to the end directory task queue. + * If the file or directory that was just queued has children, + * try to queue tasks for the children. Each child task will go + * into either the directory queue if it is a directory, or + * directly into the queue for the file ingest threads, if it + * passes the filter for the job. */ try { for (Content child : file.getChildren()) { @@ -193,10 +194,9 @@ final class IngestTasksScheduler { AbstractFile childFile = (AbstractFile) child; FileIngestTask childTask = new FileIngestTask(job, childFile); if (childFile.hasChildren()) { - this.directoryFileTasks.add(childTask); + this.directoryFileTaskQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - this.activeFileTasks.addLast(childTask); - newTasksForIngestThreads.addFirst(childTask); + newTasks.add(task); } } } @@ -206,15 +206,18 @@ final class IngestTasksScheduler { } /* - * Add the newly active tasks into the queue for the file ingest - * threads, AFTER the higher priority tasks that are already queued. + * The files are added to the queue for the ingest threads BEFORE + * the other queued tasks because the primary use case for this + * method is adding derived files from a higher priority task that + * preceded the tasks currently in the queue. */ - for (FileIngestTask newTask : newTasksForIngestThreads) { + for (FileIngestTask newTask : newTasks) { try { - this.fileTaskQueue.tasks.putLast(newTask); + this.queuedAndRunningFileTasks.add(newTask); + this.fileTaskQueueForIngestThreads.addFirst(newTask); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while data source ingest thread blocked on a full queue", ex); // RJCTODO: Should this propagate? Correct message - this.activeFileTasks.remove(newTask); + this.queuedAndRunningFileTasks.remove(newTask); + IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while blocked on a full file ingest threads queue", ex); Thread.currentThread().interrupt(); break; } @@ -229,46 +232,45 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(DataSourceIngestTask task) { - this.activeDataSourceTasks.remove(task); - shuffleFileTaskQueues(); + this.queuedAndRunningDataSourceTasks.remove(task); } /** * Allows an ingest thread to notify this ingest task scheduler that a file * level task has been completed. * - * @param task + * @param task The completed task. */ synchronized void notifyTaskCompleted(FileIngestTask task) { - this.activeFileTasks.remove(task); + this.queuedAndRunningFileTasks.remove(task); shuffleFileTaskQueues(); } /** * Queries the task scheduler to determine whether or not all of the ingest - * tasks for an ingest job have been completed. + * tasks for a data source ingest job have been completed. * - * @param job The data source ingest job + * @param job The data source ingest job. * * @return True or false. */ synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) { - return !hasTasksForJob(this.activeDataSourceTasks, job) - && !hasTasksForJob(this.rootFileTasks, job) - && !hasTasksForJob(this.directoryFileTasks, job) - && !hasTasksForJob(this.activeFileTasks, job); + return !hasTasksForJob(this.queuedAndRunningDataSourceTasks, job) + && !hasTasksForJob(this.rootFileTaskQueue, job) + && !hasTasksForJob(this.directoryFileTaskQueue, job) + && !hasTasksForJob(this.queuedAndRunningFileTasks, job); } /** * Clears the "upstream" task scheduling queues for a data source ingest - * job, but does nothing about tasks that have already been activated, i.e., - * moved into the queue that is consumed by the file ingest threads. + * job, but does nothing about tasks that have already been moved into the + * queue that is consumed by the file ingest threads. * - * @param job The data source ingest job + * @param job The data source ingest job. */ synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { - this.removeTasksForJob(this.rootFileTasks, job); - this.removeTasksForJob(this.directoryFileTasks, job); + this.removeTasksForJob(this.rootFileTaskQueue, job); + this.removeTasksForJob(this.directoryFileTaskQueue, job); } /** @@ -276,8 +278,9 @@ final class IngestTasksScheduler { * files and virtual directories for a data source. Used to create file * tasks to put into the root directories queue. * - * @param dataSource The data source. - * @param topLevelFiles The top level files are added to this list. + * @param dataSource The data source. + * + * @return The top level files. */ private static List getTopLevelFiles(Content dataSource) { List topLevelFiles = new ArrayList<>(); @@ -312,42 +315,53 @@ final class IngestTasksScheduler { } /** - * Intelligently queues file ingest tasks for the ingest manager's file - * ingest threads by "shuffling" them through a sequence of queues that - * allows for the interleaving of tasks from different data source ingest - * jobs based on priority. The sequence of queues is: + * Schedules file ingest tasks for the ingest manager's file ingest threads + * by "shuffling" them through a sequence of three queues that allows for + * the interleaving of tasks from different data source ingest jobs based on + * priority. The sequence of queues is: * - * 1. The root file tasks priority queue, which contains the tasks for the - * roots of data source content sub trees that are being analyzed for the - * data source ingest jobs. For example, typical root tasks for a disk image - * data source would be the tasks for the contents of the root directories - * of the file systems. This queue is a priority queue that attempts to - * ensure that user directory content is analyzed before general file system - * content. It feeds into the directory tasks queue. + * 1. The root file tasks priority queue, which contains file tasks for the + * root objects of the data sources that are being analyzed. For example, + * the root tasks for a disk image data source are typically the tasks for + * the contents of the root directories of the file systems. This queue is a + * priority queue that attempts to ensure that user directory content is + * analyzed before general file system content. It feeds into the directory + * tasks queue. * - * 2. The directory file tasks queue, which contains directory tasks - * discovered in the descent through the content sub trees that are being - * analyzed for the data source ingest jobs. It feeds into the active tasks - * queue. + * 2. The directory file tasks queue, which contains root file tasks + * shuffled out of the root tasks queue, plus directory tasks discovered in + * the descent from the root tasks to the final leaf tasks in the content + * trees that are being analyzed for the data source ingest jobs. This queue + * is a FIFO queue. It feeds into the file tasks queue for the ingest + * manager's file ingest threads. * - * 3. The active file tasks queue, a queue of the file tasks that are either - * in the tasks queue for the file ingest threads or are in the process of - * being analyzed in a file ingest thread. + * 3. The file tasks queue for the ingest manager's file ingest threads. + * This queue is a blocking deque that is FIFO during a shuffle to maintain + * task prioritization, but LIFO when adding derived files to it directly + * during ingest. The reason for the LIFO additions is to give priority + * derived files of priority files. * - * 4. The file tasks queue for the ingest manager's file ingest threads. + * There is a fourth collection of file tasks, a "tracking" list, that keeps + * track of the file tasks that are either in the tasks queue for the file + * ingest threads, or are in the process of being analyzed in a file ingest + * thread. This queue is vital to the ingest task scheduler's ability to + * determine when all of the ingest tasks for a data source ingest job have + * been completed. It is also used to drive this shuffling algorithm - + * whenever this list is empty, the two "upstream" queues are "shuffled" to + * queue more tasks for the file ingest threads. */ synchronized private void shuffleFileTaskQueues() { - final Deque newTasksForIngestThreads = new LinkedList<>(); - while (this.activeFileTasks.isEmpty()) { + List newTasks = new LinkedList<>(); + while (this.queuedAndRunningFileTasks.isEmpty()) { /* * If the directory file task queue is empty, move the highest * priority root file task, if there is one, into it. If both the - * root and the directory file task queuess are empty, there is - * nothing left to do. + * root and the directory file task queues are empty, there is + * nothing left to shuffle, so exit. */ - if (this.directoryFileTasks.isEmpty()) { - if (!this.rootFileTasks.isEmpty()) { - this.directoryFileTasks.add(this.rootFileTasks.pollFirst()); + if (this.directoryFileTaskQueue.isEmpty()) { + if (!this.rootFileTaskQueue.isEmpty()) { + this.directoryFileTaskQueue.add(this.rootFileTaskQueue.pollFirst()); } else { return; } @@ -355,21 +369,22 @@ final class IngestTasksScheduler { /* * Try to move the next task from the directory task queue into the - * active file tasks tracking queue, if it passes the filter for the - * job. + * queue for the file ingest threads, if it passes the filter for + * the job. The file is added to the queue for the ingest threads + * AFTER the higher priority tasks that preceded it. */ - final FileIngestTask directoryTask = this.directoryFileTasks.pollLast(); + final FileIngestTask directoryTask = this.directoryFileTaskQueue.pollLast(); if (shouldEnqueueFileTask(directoryTask)) { - this.activeFileTasks.addFirst(directoryTask); - newTasksForIngestThreads.addFirst(directoryTask); + newTasks.add(directoryTask); } /* - * If the file or directory from the next that was just activated - * has children, try to queue tasks for the children. Each child - * will go into the directory task queue if it is a directory, or - * into the active file tasks tracking queue if it passes the filter - * for the job. + * If the directory (or root level file) that was just queued has + * children, try to queue tasks for the children. Each child task + * will go into either the directory queue if it is a directory, or + * into the queue for the file ingest threads, if it passes the + * filter for the job. The file is added to the queue for the ingest + * threads AFTER the higher priority tasks that preceded it. */ final AbstractFile directory = directoryTask.getFile(); try { @@ -378,38 +393,30 @@ final class IngestTasksScheduler { AbstractFile childFile = (AbstractFile) child; FileIngestTask childTask = new FileIngestTask(directoryTask.getIngestJob(), childFile); if (childFile.hasChildren()) { - this.directoryFileTasks.add(childTask); + this.directoryFileTaskQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - this.activeFileTasks.add(directoryTask); - /* - * Queue the child file tasks for the ingest threads - * in front of parent directory tasks. - */ - newTasksForIngestThreads.addFirst(directoryTask); + newTasks.add(childTask); } } } } catch (TskCoreException ex) { logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", directory.getName(), directory.getId()), ex); //NON-NLS } + } - /* - * Add the newly active tasks into the queue for the file ingest - * threads, AFTER the higher priority tasks that are already queued. - */ - for (FileIngestTask newTask : newTasksForIngestThreads) { - try { - this.fileTaskQueue.tasks.putLast(newTask); - } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a - * full queue. Discard the task and reset the interrupted - * flag. - */ - IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while data source ingest thread blocked on a full queue", ex); // RJCTODO: Should this propagate? Correct message - this.activeFileTasks.remove(newTask); - Thread.currentThread().interrupt(); - } + /* + * The files are added to the queue for the ingest threads AFTER the + * higher priority tasks that preceded them. + */ + for (FileIngestTask newTask : newTasks) { + try { + this.queuedAndRunningFileTasks.add(newTask); + this.fileTaskQueueForIngestThreads.addFirst(newTask); + } catch (InterruptedException ex) { + this.queuedAndRunningFileTasks.remove(newTask); + IngestTasksScheduler.logger.log(Level.INFO, "Ingest cancelled while blocked on a full file ingest threads queue", ex); + Thread.currentThread().interrupt(); + break; } } } @@ -674,32 +681,40 @@ final class IngestTasksScheduler { } /** - * Wraps access to pending data source ingest tasks in the interface - * required by the ingest threads. + * A blocking queue of data source ingest tasks for the ingest manager's + * data source ingest threads. */ private final class DataSourceIngestTaskQueue implements IngestTaskQueue { private final BlockingQueue tasks = new LinkedBlockingQueue<>(); + private void add(DataSourceIngestTask task) throws InterruptedException { + this.tasks.put(task); + } + @Override public IngestTask getNextTask() throws InterruptedException { return tasks.take(); } - private void add(DataSourceIngestTask task) throws InterruptedException { - this.tasks.put(task); - } - } /** - * Wraps access to pending file ingest tasks in the interface required by - * the ingest threads. + * A blocking, LIFO queue of data source ingest tasks for the ingest + * manager's data source ingest threads. */ private final class FileIngestTaskQueue implements IngestTaskQueue { private final BlockingDeque tasks = new LinkedBlockingDeque<>(); + private void addFirst(FileIngestTask task) throws InterruptedException { + this.tasks.putFirst(task); + } + + private void addLast(FileIngestTask task) throws InterruptedException { + this.tasks.putLast(task); + } + @Override public IngestTask getNextTask() throws InterruptedException { return tasks.takeFirst(); @@ -726,11 +741,11 @@ final class IngestTasksScheduler { */ IngestJobTasksSnapshot(long jobId) { this.jobId = jobId; - this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootFileTasks, jobId); - this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.directoryFileTasks, jobId); - this.fileQueueSize = countTasksForJob(IngestTasksScheduler.this.fileTaskQueue.tasks, jobId); - this.dsQueueSize = countTasksForJob(IngestTasksScheduler.this.dataSourceTaskQueue.tasks, jobId); - this.runningListSize = countTasksForJob(IngestTasksScheduler.this.activeDataSourceTasks, jobId) + countTasksForJob(IngestTasksScheduler.this.activeFileTasks, jobId); + this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootFileTaskQueue, jobId); + this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.directoryFileTaskQueue, jobId); + this.fileQueueSize = countTasksForJob(IngestTasksScheduler.this.fileTaskQueueForIngestThreads.tasks, jobId); + this.dsQueueSize = countTasksForJob(IngestTasksScheduler.this.dataSourceTaskQueueForIngestThreads.tasks, jobId); + this.runningListSize = countTasksForJob(IngestTasksScheduler.this.queuedAndRunningDataSourceTasks, jobId) + countTasksForJob(IngestTasksScheduler.this.queuedAndRunningFileTasks, jobId); } /** From 75d4b9ad5502f28be15e85f902c8cbbfb8fa1ac5 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 19 Mar 2018 19:46:01 -0400 Subject: [PATCH 50/50] Fix IngestTasksScheduler, IngestManager concurrency issues --- .../autopsy/ingest/IngestTasksScheduler.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 2bc1769bc6..36b09035b1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -166,7 +166,7 @@ final class IngestTasksScheduler { */ synchronized void scheduleFileIngestTasks(DataSourceIngestJob job, Collection files) { if (!job.isCancelled()) { - List newTasks = new LinkedList<>(); + List newTasksForFileIngestThreads = new LinkedList<>(); for (AbstractFile file : files) { /* * Put the file directly into the queue for the file ingest @@ -178,7 +178,7 @@ final class IngestTasksScheduler { */ FileIngestTask task = new FileIngestTask(job, file); if (shouldEnqueueFileTask(task)) { - newTasks.add(task); + newTasksForFileIngestThreads.add(task); } /* @@ -196,7 +196,7 @@ final class IngestTasksScheduler { if (childFile.hasChildren()) { this.directoryFileTaskQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - newTasks.add(task); + newTasksForFileIngestThreads.add(task); } } } @@ -211,7 +211,7 @@ final class IngestTasksScheduler { * method is adding derived files from a higher priority task that * preceded the tasks currently in the queue. */ - for (FileIngestTask newTask : newTasks) { + for (FileIngestTask newTask : newTasksForFileIngestThreads) { try { this.queuedAndRunningFileTasks.add(newTask); this.fileTaskQueueForIngestThreads.addFirst(newTask); @@ -351,7 +351,7 @@ final class IngestTasksScheduler { * queue more tasks for the file ingest threads. */ synchronized private void shuffleFileTaskQueues() { - List newTasks = new LinkedList<>(); + List newTasksForFileIngestThreads = new LinkedList<>(); while (this.queuedAndRunningFileTasks.isEmpty()) { /* * If the directory file task queue is empty, move the highest @@ -375,7 +375,8 @@ final class IngestTasksScheduler { */ final FileIngestTask directoryTask = this.directoryFileTaskQueue.pollLast(); if (shouldEnqueueFileTask(directoryTask)) { - newTasks.add(directoryTask); + newTasksForFileIngestThreads.add(directoryTask); + this.queuedAndRunningFileTasks.add(directoryTask); } /* @@ -395,7 +396,8 @@ final class IngestTasksScheduler { if (childFile.hasChildren()) { this.directoryFileTaskQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - newTasks.add(childTask); + newTasksForFileIngestThreads.add(childTask); + this.queuedAndRunningFileTasks.add(childTask); } } } @@ -408,9 +410,8 @@ final class IngestTasksScheduler { * The files are added to the queue for the ingest threads AFTER the * higher priority tasks that preceded them. */ - for (FileIngestTask newTask : newTasks) { + for (FileIngestTask newTask : newTasksForFileIngestThreads) { try { - this.queuedAndRunningFileTasks.add(newTask); this.fileTaskQueueForIngestThreads.addFirst(newTask); } catch (InterruptedException ex) { this.queuedAndRunningFileTasks.remove(newTask);