From a19b32e3893278034f22b9cd1b925abc45a27e2e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 1 Nov 2017 16:25:15 -0400 Subject: [PATCH 1/4] Rewrote portions of 'doRecoveryIfCrashed()' to fix bugs. --- .../autoingest/AutoIngestManager.java | 248 +++++------------- 1 file changed, 63 insertions(+), 185 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 9f43879b55..a779085813 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -37,7 +37,6 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -59,13 +58,11 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.Immutable; -import javax.annotation.concurrent.ThreadSafe; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.CaseActionException; +import org.sleuthkit.autopsy.casemodule.CaseDetails; import org.sleuthkit.autopsy.casemodule.CaseMetadata; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; @@ -101,7 +98,6 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; -import org.sleuthkit.datamodel.Content; /** * An auto ingest manager is responsible for processing auto ingest jobs defined @@ -503,7 +499,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang } SYS_LOGGER.log(Level.INFO, "Starting input scan of {0}", rootInputDirectory); InputDirScanner scanner = new InputDirScanner(); - scanner.scan(); SYS_LOGGER.log(Level.INFO, "Completed input scan of {0}", rootInputDirectory); } @@ -559,12 +554,10 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (!prioritizedJobs.isEmpty()) { ++maxPriority; for (AutoIngestJob job : prioritizedJobs) { - int oldPriority = job.getPriority(); - job.setPriority(maxPriority); try { this.updateCoordinationServiceNode(job); + job.setPriority(maxPriority); } catch (CoordinationServiceException | InterruptedException ex) { - job.setPriority(oldPriority); throw new AutoIngestManagerException("Error updating case priority", ex); } } @@ -615,14 +608,12 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ if (null != prioritizedJob) { ++maxPriority; - int oldPriority = prioritizedJob.getPriority(); - prioritizedJob.setPriority(maxPriority); try { this.updateCoordinationServiceNode(prioritizedJob); } catch (CoordinationServiceException | InterruptedException ex) { - prioritizedJob.setPriority(oldPriority); throw new AutoIngestManagerException("Error updating job priority", ex); } + prioritizedJob.setPriority(maxPriority); } Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator()); @@ -1051,8 +1042,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (null != manifest) { /* - * Update the mapping of case names to manifest paths that - * is used for case deletion. + * Update the mapping of case names to manifest paths that is + * used for case deletion. */ String caseName = manifest.getCaseName(); Path manifestPath = manifest.getFilePath(); @@ -1066,8 +1057,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /* - * Add a job to the pending jobs queue, the completed jobs - * list, or do crashed job recovery, as required. + * Add a job to the pending jobs queue, the completed jobs list, + * or do crashed job recovery, as required. */ try { byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()); @@ -1087,7 +1078,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang break; case DELETED: /* - * Ignore jobs marked as "deleted." + * Ignore jobs marked as "deleted." */ break; default: @@ -1247,48 +1238,38 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (null != manifestLock) { SYS_LOGGER.log(Level.SEVERE, "Attempting crash recovery for {0}", manifestPath); try { + Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); + /* * Create the recovery job. */ AutoIngestJob job = new AutoIngestJob(nodeData); int numberOfCrashes = job.getNumberOfCrashes(); - ++numberOfCrashes; - job.setNumberOfCrashes(numberOfCrashes); - job.setCompletedDate(new Date(0)); - Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { + ++numberOfCrashes; + job.setNumberOfCrashes(numberOfCrashes); + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { + job.setCompletedDate(new Date(0)); + } else { + job.setCompletedDate(Date.from(Instant.now())); + } + } + if (null != caseDirectoryPath) { job.setCaseDirectoryPath(caseDirectoryPath); job.setErrorsOccurred(true); - } else { - job.setErrorsOccurred(false); - } - - /* - * Update the coordination service node for the job. If - * this fails, leave the recovery to another host. - */ - try { - updateCoordinationServiceNode(job); - if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { - newPendingJobsList.add(job); - } else { - newCompletedJobsList.add(new AutoIngestJob(nodeData)); - } - } catch (CoordinationServiceException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex); - return; - } - - /* - * Write the alert file and do the logging. - */ - if (null != caseDirectoryPath) { try { - AutoIngestAlertFile.create(nodeData.getCaseDirectoryPath()); + /* + * Write the alert file and do the logging. + */ + AutoIngestAlertFile.create(caseDirectoryPath); } catch (AutoIngestAlertFileException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error creating alert file for crashed job for %s", manifestPath), ex); } + } else { + job.setErrorsOccurred(false); } + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { job.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING); if (null != caseDirectoryPath) { @@ -1302,13 +1283,32 @@ public final class AutoIngestManager extends Observable implements PropertyChang job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); if (null != caseDirectoryPath) { try { - new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), nodeData.getCaseDirectoryPath()).logCrashRecoveryNoRetry(); + new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryNoRetry(); } catch (AutoIngestJobLoggerException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error creating case auto ingest log entry for crashed job for %s", manifestPath), ex); } } } + /* + * Update the coordination service node for the job. If + * this fails, leave the recovery to another host. + */ + try { + updateCoordinationServiceNode(job); + } catch (CoordinationServiceException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex); + return; + } + + nodeData = new AutoIngestJobNodeData(job); + + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { + newPendingJobsList.add(job); + } else { + newCompletedJobsList.add(new AutoIngestJob(nodeData)); + } + } finally { try { manifestLock.release(); @@ -1459,7 +1459,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ private final class JobProcessingTask implements Runnable { - private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; private final Object ingestLock; private final Object pauseLock; @GuardedBy("pauseLock") @@ -2126,7 +2125,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang Case.openAsCurrentCase(metadataFilePath.toString()); } else { caseDirectoryPath = PathUtils.createCaseFolderPath(rootOutputDirectory, caseName); - Case.createAsCurrentCase(caseDirectoryPath.toString(), caseName, "", "", CaseType.MULTI_USER_CASE); + CaseDetails caseDetails = new CaseDetails(caseName, "", "", "", "", ""); + Case.createAsCurrentCase(CaseType.MULTI_USER_CASE, caseDirectoryPath.toString(), caseDetails); /* * Sleep a bit before releasing the lock to ensure * that the new case folder is visible on the @@ -2222,7 +2222,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang return; } - DataSource dataSource = identifyDataSource(caseForJob); + DataSource dataSource = identifyDataSource(); if (null == dataSource) { currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now())); return; @@ -2275,7 +2275,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * interrupted while blocked, i.e., * if auto ingest is shutting down. */ - private DataSource identifyDataSource(Case caseForJob) throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private DataSource identifyDataSource() throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath); @@ -2294,7 +2294,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang String deviceId = manifest.getDeviceId(); return new DataSource(deviceId, dataSourcePath); } - + /** * Passes the data source for the current job through a data source * processor that adds it to the case database. @@ -2317,28 +2317,21 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.INFO, "Adding data source for {0} ", manifestPath); currentJob.setProcessingStage(AutoIngestJob.Stage.ADDING_DATA_SOURCE, Date.from(Instant.now())); UUID taskId = UUID.randomUUID(); - DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId); + DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId, ingestLock); DataSourceProcessorProgressMonitor progressMonitor = new DoNothingDSPProgressMonitor(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, manifest.getDataSourceFileName(), caseDirectoryPath); try { caseForJob.notifyAddingDataSource(taskId); - // lookup all AutomatedIngestDataSourceProcessors - Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); - - Map validDataSourceProcessorsMap = new HashMap<>(); - for (AutoIngestDataSourceProcessor processor : processorCandidates) { - try { - int confidence = processor.canProcess(dataSource.getPath()); - if (confidence > 0) { - validDataSourceProcessorsMap.put(processor, confidence); - } - } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { - SYS_LOGGER.log(Level.SEVERE, "Exception while determining whether data source processor {0} can process {1}", new Object[]{processor.getDataSourceType(), dataSource.getPath()}); - // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. - throw ex; - } + Map validDataSourceProcessorsMap; + try { + // lookup all AutomatedIngestDataSourceProcessors and poll which ones are able to process the current data source + validDataSourceProcessorsMap = DataSourceProcessorUtility.getDataSourceProcessor(dataSource.getPath()); + } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { + SYS_LOGGER.log(Level.SEVERE, "Exception while determining best data source processor for {0}", dataSource.getPath()); + // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. + throw ex; } // did we find a data source processor that can process the data source @@ -2595,80 +2588,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang jobLogger.logFileExportError(); } } - - /** - * A "callback" that collects the results of running a data source - * processor on a data source and unblocks the job processing thread - * when the data source processor finishes running in its own thread. - */ - @Immutable - class AddDataSourceCallback extends DataSourceProcessorCallback { - - private final Case caseForJob; - private final DataSource dataSourceInfo; - private final UUID taskId; - - /** - * Constructs a "callback" that collects the results of running a - * data source processor on a data source and unblocks the job - * processing thread when the data source processor finishes running - * in its own thread. - * - * @param caseForJob The case for the current job. - * @param dataSourceInfo The data source - * @param taskId The task id to associate with ingest job - * events. - */ - AddDataSourceCallback(Case caseForJob, DataSource dataSourceInfo, UUID taskId) { - this.caseForJob = caseForJob; - this.dataSourceInfo = dataSourceInfo; - this.taskId = taskId; - } - - /** - * Called by the data source processor when it finishes running in - * its own thread. - * - * @param result The result code for the processing of - * the data source. - * @param errorMessages Any error messages generated during the - * processing of the data source. - * @param dataSourceContent The content produced by processing the - * data source. - */ - @Override - public void done(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSourceContent) { - if (!dataSourceContent.isEmpty()) { - caseForJob.notifyDataSourceAdded(dataSourceContent.get(0), taskId); - } else { - caseForJob.notifyFailedAddingDataSource(taskId); - } - dataSourceInfo.setDataSourceProcessorOutput(result, errorMessages, dataSourceContent); - dataSourceContent.addAll(dataSourceContent); - synchronized (ingestLock) { - ingestLock.notify(); - } - } - - /** - * Called by the data source processor when it finishes running in - * its own thread, if that thread is the AWT (Abstract Window - * Toolkit) event dispatch thread (EDT). - * - * @param result The result code for the processing of - * the data source. - * @param errorMessages Any error messages generated during the - * processing of the data source. - * @param dataSourceContent The content produced by processing the - * data source. - */ - @Override - public void doneEDT(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSources) { - done(result, errorMessages, dataSources); - } - - } - + /** * A data source processor progress monitor does nothing. There is * currently no mechanism for showing or recording data source processor @@ -3008,49 +2928,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang PARTIALLY_DELETED, FULLY_DELETED } - - @ThreadSafe - private static final class DataSource { - - private final String deviceId; - private final Path path; - private DataSourceProcessorResult resultCode; - private List errorMessages; - private List content; - - DataSource(String deviceId, Path path) { - this.deviceId = deviceId; - this.path = path; - } - - String getDeviceId() { - return deviceId; - } - - Path getPath() { - return this.path; - } - - synchronized void setDataSourceProcessorOutput(DataSourceProcessorResult result, List errorMessages, List content) { - this.resultCode = result; - this.errorMessages = new ArrayList<>(errorMessages); - this.content = new ArrayList<>(content); - } - - synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode() { - return resultCode; - } - - synchronized List getDataSourceProcessorErrorMessages() { - return new ArrayList<>(errorMessages); - } - - synchronized List getContent() { - return new ArrayList<>(content); - } - - } - + static final class AutoIngestManagerException extends Exception { private static final long serialVersionUID = 1L; From 29c25d71e6cbd59a64831a6ee0997091b81261cd Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 1 Nov 2017 16:35:44 -0400 Subject: [PATCH 2/4] Minor revision --- .../autopsy/experimental/autoingest/AutoIngestManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index a779085813..cf41d00df2 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -62,7 +62,6 @@ import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.CaseActionException; -import org.sleuthkit.autopsy.casemodule.CaseDetails; import org.sleuthkit.autopsy.casemodule.CaseMetadata; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; @@ -2125,8 +2124,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang Case.openAsCurrentCase(metadataFilePath.toString()); } else { caseDirectoryPath = PathUtils.createCaseFolderPath(rootOutputDirectory, caseName); - CaseDetails caseDetails = new CaseDetails(caseName, "", "", "", "", ""); - Case.createAsCurrentCase(CaseType.MULTI_USER_CASE, caseDirectoryPath.toString(), caseDetails); + Case.createAsCurrentCase(caseDirectoryPath.toString(), caseName, "", "", CaseType.MULTI_USER_CASE); /* * Sleep a bit before releasing the lock to ensure * that the new case folder is visible on the From d0e5f9d50b7c7f5c961e0611b93d38fdcce2deee Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 1 Nov 2017 16:40:55 -0400 Subject: [PATCH 3/4] Minor revision --- .../autoingest/AutoIngestManager.java | 177 +++++++++++++++--- 1 file changed, 155 insertions(+), 22 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index cf41d00df2..3f28487394 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -37,6 +37,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -58,6 +59,9 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.ThreadSafe; +import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; @@ -97,6 +101,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; +import org.sleuthkit.datamodel.Content; /** * An auto ingest manager is responsible for processing auto ingest jobs defined @@ -498,6 +503,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang } SYS_LOGGER.log(Level.INFO, "Starting input scan of {0}", rootInputDirectory); InputDirScanner scanner = new InputDirScanner(); + scanner.scan(); SYS_LOGGER.log(Level.INFO, "Completed input scan of {0}", rootInputDirectory); } @@ -553,10 +559,12 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (!prioritizedJobs.isEmpty()) { ++maxPriority; for (AutoIngestJob job : prioritizedJobs) { + int oldPriority = job.getPriority(); + job.setPriority(maxPriority); try { this.updateCoordinationServiceNode(job); - job.setPriority(maxPriority); } catch (CoordinationServiceException | InterruptedException ex) { + job.setPriority(oldPriority); throw new AutoIngestManagerException("Error updating case priority", ex); } } @@ -607,12 +615,14 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ if (null != prioritizedJob) { ++maxPriority; + int oldPriority = prioritizedJob.getPriority(); + prioritizedJob.setPriority(maxPriority); try { this.updateCoordinationServiceNode(prioritizedJob); } catch (CoordinationServiceException | InterruptedException ex) { + prioritizedJob.setPriority(oldPriority); throw new AutoIngestManagerException("Error updating job priority", ex); } - prioritizedJob.setPriority(maxPriority); } Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator()); @@ -1041,8 +1051,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (null != manifest) { /* - * Update the mapping of case names to manifest paths that is - * used for case deletion. + * Update the mapping of case names to manifest paths that + * is used for case deletion. */ String caseName = manifest.getCaseName(); Path manifestPath = manifest.getFilePath(); @@ -1056,8 +1066,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /* - * Add a job to the pending jobs queue, the completed jobs list, - * or do crashed job recovery, as required. + * Add a job to the pending jobs queue, the completed jobs + * list, or do crashed job recovery, as required. */ try { byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()); @@ -1077,7 +1087,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang break; case DELETED: /* - * Ignore jobs marked as "deleted." + * Ignore jobs marked as "deleted." */ break; default: @@ -1458,6 +1468,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ private final class JobProcessingTask implements Runnable { + private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; private final Object ingestLock; private final Object pauseLock; @GuardedBy("pauseLock") @@ -2220,7 +2231,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang return; } - DataSource dataSource = identifyDataSource(); + DataSource dataSource = identifyDataSource(caseForJob); if (null == dataSource) { currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now())); return; @@ -2273,7 +2284,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * interrupted while blocked, i.e., * if auto ingest is shutting down. */ - private DataSource identifyDataSource() throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private DataSource identifyDataSource(Case caseForJob) throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath); @@ -2292,7 +2303,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang String deviceId = manifest.getDeviceId(); return new DataSource(deviceId, dataSourcePath); } - + /** * Passes the data source for the current job through a data source * processor that adds it to the case database. @@ -2315,21 +2326,28 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.INFO, "Adding data source for {0} ", manifestPath); currentJob.setProcessingStage(AutoIngestJob.Stage.ADDING_DATA_SOURCE, Date.from(Instant.now())); UUID taskId = UUID.randomUUID(); - DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId, ingestLock); + DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId); DataSourceProcessorProgressMonitor progressMonitor = new DoNothingDSPProgressMonitor(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, manifest.getDataSourceFileName(), caseDirectoryPath); try { caseForJob.notifyAddingDataSource(taskId); - Map validDataSourceProcessorsMap; - try { - // lookup all AutomatedIngestDataSourceProcessors and poll which ones are able to process the current data source - validDataSourceProcessorsMap = DataSourceProcessorUtility.getDataSourceProcessor(dataSource.getPath()); - } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { - SYS_LOGGER.log(Level.SEVERE, "Exception while determining best data source processor for {0}", dataSource.getPath()); - // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. - throw ex; + // lookup all AutomatedIngestDataSourceProcessors + Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); + + Map validDataSourceProcessorsMap = new HashMap<>(); + for (AutoIngestDataSourceProcessor processor : processorCandidates) { + try { + int confidence = processor.canProcess(dataSource.getPath()); + if (confidence > 0) { + validDataSourceProcessorsMap.put(processor, confidence); + } + } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { + SYS_LOGGER.log(Level.SEVERE, "Exception while determining whether data source processor {0} can process {1}", new Object[]{processor.getDataSourceType(), dataSource.getPath()}); + // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. + throw ex; + } } // did we find a data source processor that can process the data source @@ -2586,7 +2604,80 @@ public final class AutoIngestManager extends Observable implements PropertyChang jobLogger.logFileExportError(); } } - + + /** + * A "callback" that collects the results of running a data source + * processor on a data source and unblocks the job processing thread + * when the data source processor finishes running in its own thread. + */ + @Immutable + class AddDataSourceCallback extends DataSourceProcessorCallback { + + private final Case caseForJob; + private final DataSource dataSourceInfo; + private final UUID taskId; + + /** + * Constructs a "callback" that collects the results of running a + * data source processor on a data source and unblocks the job + * processing thread when the data source processor finishes running + * in its own thread. + * + * @param caseForJob The case for the current job. + * @param dataSourceInfo The data source + * @param taskId The task id to associate with ingest job + * events. + */ + AddDataSourceCallback(Case caseForJob, DataSource dataSourceInfo, UUID taskId) { + this.caseForJob = caseForJob; + this.dataSourceInfo = dataSourceInfo; + this.taskId = taskId; + } + + /** + * Called by the data source processor when it finishes running in + * its own thread. + * + * @param result The result code for the processing of + * the data source. + * @param errorMessages Any error messages generated during the + * processing of the data source. + * @param dataSourceContent The content produced by processing the + * data source. + */ + @Override + public void done(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSourceContent) { + if (!dataSourceContent.isEmpty()) { + caseForJob.notifyDataSourceAdded(dataSourceContent.get(0), taskId); + } else { + caseForJob.notifyFailedAddingDataSource(taskId); + } + dataSourceInfo.setDataSourceProcessorOutput(result, errorMessages, dataSourceContent); + dataSourceContent.addAll(dataSourceContent); + synchronized (ingestLock) { + ingestLock.notify(); + } + } + + /** + * Called by the data source processor when it finishes running in + * its own thread, if that thread is the AWT (Abstract Window + * Toolkit) event dispatch thread (EDT). + * + * @param result The result code for the processing of + * the data source. + * @param errorMessages Any error messages generated during the + * processing of the data source. + * @param dataSourceContent The content produced by processing the + * data source. + */ + @Override + public void doneEDT(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSources) { + done(result, errorMessages, dataSources); + } + + } + /** * A data source processor progress monitor does nothing. There is * currently no mechanism for showing or recording data source processor @@ -2926,7 +3017,49 @@ public final class AutoIngestManager extends Observable implements PropertyChang PARTIALLY_DELETED, FULLY_DELETED } - + + @ThreadSafe + private static final class DataSource { + + private final String deviceId; + private final Path path; + private DataSourceProcessorResult resultCode; + private List errorMessages; + private List content; + + DataSource(String deviceId, Path path) { + this.deviceId = deviceId; + this.path = path; + } + + String getDeviceId() { + return deviceId; + } + + Path getPath() { + return this.path; + } + + synchronized void setDataSourceProcessorOutput(DataSourceProcessorResult result, List errorMessages, List content) { + this.resultCode = result; + this.errorMessages = new ArrayList<>(errorMessages); + this.content = new ArrayList<>(content); + } + + synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode() { + return resultCode; + } + + synchronized List getDataSourceProcessorErrorMessages() { + return new ArrayList<>(errorMessages); + } + + synchronized List getContent() { + return new ArrayList<>(content); + } + + } + static final class AutoIngestManagerException extends Exception { private static final long serialVersionUID = 1L; @@ -2941,4 +3074,4 @@ public final class AutoIngestManager extends Observable implements PropertyChang } -} +} \ No newline at end of file From 536056f415b018924a35114b03f1adc1989ccd2a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 1 Nov 2017 16:49:49 -0400 Subject: [PATCH 4/4] Removed unused import and variable. --- .../autopsy/experimental/autoingest/AutoIngestManager.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 3f28487394..c169223396 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -61,7 +61,6 @@ import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; @@ -122,7 +121,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang private static final int NUM_INPUT_SCAN_SCHEDULING_THREADS = 1; private static final String INPUT_SCAN_SCHEDULER_THREAD_NAME = "AIM-input-scan-scheduler-%d"; private static final String INPUT_SCAN_THREAD_NAME = "AIM-input-scan-%d"; - private static int DEFAULT_JOB_PRIORITY = 0; private static final String AUTO_INGEST_THREAD_NAME = "AIM-job-processing-%d"; private static final String LOCAL_HOST_NAME = NetworkUtils.getLocalHostName(); private static final String EVENT_CHANNEL_NAME = "Auto-Ingest-Manager-Events";