From f71063eb569700628e789a2c9414631c3dd28cf7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 10 Dec 2014 15:34:40 -0500 Subject: [PATCH 1/7] Improve INgestManager and IngestJob documentation --- .../sleuthkit/autopsy/ingest/IngestJob.java | 58 +++++++++---------- .../autopsy/ingest/IngestManager.java | 9 +-- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 98dbbb3930..ff463321a7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -23,7 +23,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.Level; import javax.swing.JOptionPane; @@ -44,7 +43,7 @@ final class IngestJob { private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); /** - * These fields define the ingest job: a unique ID supplied by the ingest + * These fields define an ingest job: a unique ID supplied by the ingest * manager, the user's ingest job settings, and the data source to be * processed. */ @@ -80,9 +79,9 @@ final class IngestJob { private final Object stageCompletionCheckLock; /** - * There is a data source level ingest modules pipeline for both the first - * stage and the second stage. Longer running, lower priority data source - * level ingest modules belong in the second stage pipeline. + * An ingest job has separate data source level ingest module pipelines for + * each processing stage. Longer running, lower priority modules belong in + * the second stage pipeline. */ private final Object dataSourceIngestPipelineLock; private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; @@ -90,27 +89,34 @@ final class IngestJob { private DataSourceIngestPipeline currentDataSourceIngestPipeline; /** - * There is a collection of identical file level ingest module pipelines, - * one for each file level ingest thread in the ingest manager. + * An ingest job has a collection of identical file level ingest module + * pipelines, one for each file level ingest thread in the ingest manager. */ - private final LinkedBlockingQueue fileIngestPipelines; - + private final LinkedBlockingQueue fileIngestPipelines; + /** - * The task scheduler singleton is responsible for creating and scheduling - * the ingest tasks that make up this ingest jobs. + * An ingest job supports cancellation of either the currently running data + * source level ingest module or the entire ingest job. + */ + private volatile boolean currentDataSourceIngestModuleCancelled; + private volatile boolean cancelled; + + /** + * An ingest job uses the task scheduler singleton to create and queue the + * ingest tasks that make up the job. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - + /** - * These fields are used to provide data source level task progress bars for - * the job. + * These fields are used to report data source level ingest progress for the + * ingest job. */ private final Object dataSourceIngestProgressLock; private ProgressHandle dataSourceIngestProgress; /** - * These fields are used to provide file level ingest task progress bars for - * the job. + * These fields are used to report file level ingest task progress for the + * ingest job. */ private final Object fileIngestProgressLock; private final List filesInProgress; @@ -119,24 +125,16 @@ final class IngestJob { private ProgressHandle fileIngestProgress; /** - * These fields support cancellation of either the currently running data - * source level ingest module or the entire ingest job. + * This field is used to record the creation of the ingest job. */ - private volatile boolean currentDataSourceIngestModuleCancelled; - private volatile boolean cancelled; - - /** - * This field is used for generating ingest job diagnostic data. - */ - private final long startTime; + private final long createTime; /** * Constructs an ingest job. * * @param id The identifier assigned to the job. * @param dataSource The data source to be ingested. - * @param processUnallocatedSpace Whether or not unallocated space should be - * processed during the ingest job. + * @param settings The settings for the ingest job. */ IngestJob(long id, Content dataSource, IngestJobSettings settings) { this.id = id; @@ -149,7 +147,7 @@ final class IngestJob { this.fileIngestProgressLock = new Object(); this.stage = IngestJob.Stages.INITIALIZATION; this.stageCompletionCheckLock = new Object(); - this.startTime = new Date().getTime(); + this.createTime = new Date().getTime(); } /** @@ -934,13 +932,13 @@ final class IngestJob { private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; /** - * Constructs an object to stores basic diagnostic statistics for an + * Constructs an object to store basic diagnostic statistics for an * ingest job. */ IngestJobSnapshot() { this.jobId = IngestJob.this.id; this.dataSource = IngestJob.this.dataSource.getName(); - this.startTime = IngestJob.this.startTime; + this.startTime = IngestJob.this.createTime; synchronized (IngestJob.this.fileIngestProgressLock) { this.processedFiles = IngestJob.this.processedFiles; this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3e3f70c473..7d32adbcd3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -63,11 +63,12 @@ public class IngestManager { private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); /** - * Each runnable/callable task the ingest manager farms out to a thread pool - * is given a unique thread/task ID. + * Each runnable/callable task the ingest manager submits to its thread + * pools is given a unique thread/task ID. */ - private final AtomicLong nextThreadId = new AtomicLong(0L); - + // TODO: It is no longer necessary to have multiple thread pools. + private final AtomicLong nextThreadId = new AtomicLong(0L); + /** * Ingest jobs are started on a pool thread by ingest job starters. A * mapping of thread/task IDs to the result objects associated with each From b2cac4427459e0a542be077c31f82db540c15b02 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 12 Dec 2014 17:04:20 -0500 Subject: [PATCH 2/7] Refactoring to produce new IngestJob class, old class => DataSourceIngestJon --- .../autopsy/ingest/DataSourceIngestJob.java | 1063 ++++++++++++++++ .../DataSourceIngestModuleProgress.java | 4 +- .../ingest/DataSourceIngestPipeline.java | 116 +- .../autopsy/ingest/DataSourceIngestTask.java | 2 +- .../autopsy/ingest/FileIngestPipeline.java | 4 +- .../autopsy/ingest/FileIngestTask.java | 6 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 1081 ++--------------- .../autopsy/ingest/IngestJobContext.java | 4 +- .../autopsy/ingest/IngestManager.java | 115 +- .../ingest/IngestProgressSnapshotPanel.java | 4 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 6 +- .../autopsy/ingest/IngestTasksScheduler.java | 12 +- 12 files changed, 1355 insertions(+), 1062 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java new file mode 100644 index 0000000000..254510bb6c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -0,0 +1,1063 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import javax.swing.JOptionPane; +import org.netbeans.api.progress.ProgressHandle; +import org.netbeans.api.progress.ProgressHandleFactory; +import org.openide.util.Cancellable; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; + +/** + * Encapsulates a data source to be processed and the settings, ingest module + * pipelines, and progress bars that are used to process it. + */ +final class DataSourceIngestJob { + + private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName()); + private static final AtomicLong nextJobId = new AtomicLong(0L); + + /** + * These fields define a data source ingest job: the parent ingest job, a + * unique ID, the user's ingest job settings, and the data source to be + * processed. + */ + private final IngestJob parentJob; + private final long id; + private final IngestJobSettings settings; + private final Content dataSource; + + /** + * An ingest job runs in stages. + */ + private static enum Stages { + + /** + * Setting up for processing. + */ + INITIALIZATION, + /** + * Running high priority data source level ingest modules and file level + * ingest modules. + */ + FIRST, + /** + * Running lower priority, usually long-running, data source level + * ingest modules. + */ + SECOND, + /** + * Cleaning up. + */ + FINALIZATION + }; + private Stages stage; + private final Object stageCompletionCheckLock; + + /** + * An ingest job has separate data source level ingest module pipelines for + * each processing stage. Longer running, lower priority modules belong in + * the second stage pipeline. + */ + private final Object dataSourceIngestPipelineLock; + private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; + private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; + private DataSourceIngestPipeline currentDataSourceIngestPipeline; + + /** + * An ingest job has a collection of identical file level ingest module + * pipelines, one for each file level ingest thread in the ingest manager. + */ + private final LinkedBlockingQueue fileIngestPipelines; + + /** + * An ingest job supports cancellation of either the currently running data + * source level ingest module or the entire ingest job. + */ + private volatile boolean currentDataSourceIngestModuleCancelled; + private volatile boolean cancelled; + + /** + * An ingest job uses the task scheduler singleton to create and queue the + * ingest tasks that make up the job. + */ + private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + + /** + * These fields are used to report data source level ingest progress for the + * ingest job. + */ + private final Object dataSourceIngestProgressLock; + private ProgressHandle dataSourceIngestProgress; + + /** + * These fields are used to report file level ingest task progress for the + * ingest job. + */ + private final Object fileIngestProgressLock; + private final List filesInProgress; + private long estimatedFilesToProcess; + private long processedFiles; + private ProgressHandle fileIngestProgress; + + /** + * This field is used to record the creation of the ingest job. + */ + private final long createTime; + + /** + * Constructs an ingest job. + * + * @param parentJob The ingest job of which this data source ingest job is a + * part. + * @param dataSource The data source to be ingested. + * @param settings The settings for the ingest job. + */ + DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings) { + this.parentJob = parentJob; + this.id = DataSourceIngestJob.nextJobId.getAndIncrement(); + this.dataSource = dataSource; + this.settings = settings; + this.dataSourceIngestPipelineLock = new Object(); + this.fileIngestPipelines = new LinkedBlockingQueue<>(); + this.filesInProgress = new ArrayList<>(); + this.dataSourceIngestProgressLock = new Object(); + this.fileIngestProgressLock = new Object(); + this.stage = DataSourceIngestJob.Stages.INITIALIZATION; + this.stageCompletionCheckLock = new Object(); + this.createTime = new Date().getTime(); + } + + /** + * Gets the identifier assigned to this job. + * + * @return The job identifier. + */ + long getId() { + return this.id; + } + + /** + * Gets the data source to be ingested by this job. + * + * @return A Content object representing the data source. + */ + Content getDataSource() { + return this.dataSource; + } + + /** + * Gets whether or not unallocated space should be processed as part of this + * job. + * + * @return True or false. + */ + boolean shouldProcessUnallocatedSpace() { + return this.settings.getProcessUnallocatedSpace(); + } + + /** + * Passes the data source for this job through the currently active data + * source level ingest pipeline. + * + * @param task A data source ingest task wrapping the data source. + */ + void process(DataSourceIngestTask task) { + try { + synchronized (this.dataSourceIngestPipelineLock) { + if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + /** + * Run the data source through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } + + /** + * Shut down the data source ingest progress bar right away. Data + * source-level processing is finished for this stage. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Passes a file from the data source for this job through the file level + * ingest pipeline. + * + * @param task A file ingest task. + * @throws InterruptedException if the thread executing this code is + * interrupted while blocked on taking from or putting to the file ingest + * pipelines collection. + */ + void process(FileIngestTask task) throws InterruptedException { + try { + if (!this.isCancelled()) { + /** + * Get a file ingest pipeline not currently in use by another + * file ingest thread. + */ + FileIngestPipeline pipeline = this.fileIngestPipelines.take(); + if (!pipeline.isEmpty()) { + /** + * Get the file to process. + */ + AbstractFile file = task.getFile(); + + /** + * Update the file ingest progress bar. + */ + synchronized (this.fileIngestProgressLock) { + ++this.processedFiles; + if (this.processedFiles <= this.estimatedFilesToProcess) { + this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); + } else { + this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); + } + this.filesInProgress.add(file.getName()); + } + + /** + * Run the file through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(pipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + /** + * Update the file ingest progress bar again, in case the + * file was being displayed. + */ + if (!this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.filesInProgress.remove(file.getName()); + if (this.filesInProgress.size() > 0) { + this.fileIngestProgress.progress(this.filesInProgress.get(0)); + } else { + this.fileIngestProgress.progress(""); + } + } + } + } + + /** + * Relinquish the pipeline so it can be reused by another file + * ingest thread. + */ + this.fileIngestPipelines.put(pipeline); + } + } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Adds more files to an ingest job, i.e., extracted or carved files. Not + * currently supported for the second stage of the job. + * + * @param files A list of files to add. + */ + void addFiles(List files) { + /** + * Note: This implementation assumes that this is being called by an an + * ingest module running code on an ingest thread that is holding a + * reference to an ingest task, so no task completion check is done. + */ + if (DataSourceIngestJob.Stages.FIRST == this.stage) { + for (AbstractFile file : files) { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); + } + } else { + DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + } + + /** + * The intended clients of this method are ingest modules running code + * on an ingest thread that is holding a reference to an ingest task, in + * which case a task completion check would not be necessary. This is a + * bit of defensive programming. + */ + this.checkForStageCompleted(); + } + + /** + * Updates the display name of the data source level ingest progress bar. + * + * @param displayName The new display name. + */ + void updateDataSourceIngestProgressBarDisplayName(String displayName) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.setDisplayName(displayName); + } + } + } + + /** + * Switches the data source level ingest progress bar to determinate mode. + * This should be called if the total work units to process the data source + * is known. + * + * @param workUnits Total number of work units for the processing of the + * data source. + */ + void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToDeterminate(workUnits); + } + } + } + } + + /** + * Switches the data source level ingest progress bar to indeterminate mode. + * This should be called if the total work units to process the data source + * is unknown. + */ + void switchDataSourceIngestProgressBarToIndeterminate() { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + } + } + + /** + * Updates the data source level ingest progress bar with the number of work + * units performed, if in the determinate mode. + * + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(int workUnits) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress("", workUnits); + } + } + } + } + + /** + * Updates the data source level ingest progress with a new task name, where + * the task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + */ + void advanceDataSourceIngestProgressBar(String currentTask) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress(currentTask); + } + } + } + } + + /** + * Updates the data source level ingest progress bar with a new task name + * and the number of work units performed, if in the determinate mode. The + * task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { + if (!this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.dataSourceIngestProgress.progress(currentTask, workUnits); + } + } + } + + /** + * Queries whether or not a temporary cancellation of data source level + * ingest in order to stop the currently executing data source level ingest + * module is in effect. + * + * @return True or false. + */ + boolean currentDataSourceIngestModuleIsCancelled() { + return this.currentDataSourceIngestModuleCancelled; + } + + /** + * Rescind a temporary cancellation of data source level ingest that was + * used to stop a single data source level ingest module. + */ + void currentDataSourceIngestModuleCancellationCompleted() { + this.currentDataSourceIngestModuleCancelled = false; + + /** + * A new progress bar must be created because the cancel button of the + * previously constructed component is disabled by NetBeans when the + * user selects the "OK" button of the cancellation confirmation dialog + * popped up by NetBeans when the progress bar cancel button was + * pressed. + */ + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + this.startDataSourceIngestProgressBar(); + } + } + + /** + * Requests cancellation of ingest, i.e., a shutdown of the data source + * level and file level ingest pipelines. + */ + void cancel() { + /** + * Put a cancellation message on data source level ingest progress bar, + * if it is still running. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (dataSourceIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + dataSource.getName()); + dataSourceIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), + "IngestJob.progress.cancelling", + displayName)); + } + } + + /** + * Put a cancellation message on the file level ingest progress bar, if + * it is still running. + */ + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", + displayName)); + } + } + + this.cancelled = true; + + /** + * Tell the task scheduler to cancel all pending tasks, i.e., tasks not + * not being performed by an ingest thread. + */ + DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); + this.checkForStageCompleted(); + } + + /** + * Queries whether or not cancellation of ingest i.e., a shutdown of the + * data source level and file level ingest pipelines, has been requested. + * + * @return True or false. + */ + boolean isCancelled() { + return this.cancelled; + } + + /** + * Starts up the ingest pipelines and ingest progress bars. + * + * @return A collection of ingest module startup errors, empty on success. + */ + List start() { + this.createIngestPipelines(settings.getEnabledIngestModuleTemplates()); + List errors = startUpIngestPipelines(); + if (errors.isEmpty()) { + if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { + this.startFirstStage(); + } else if (this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } + } + return errors; + } + + /** + * Creates the file and data source ingest pipelines. + * + * @param ingestModuleTemplates Ingest module templates to use to populate + * the pipelines. + */ + private void createIngestPipelines(List ingestModuleTemplates) { + /** + * Make mappings of ingest module factory class names to templates. + */ + Map dataSourceModuleTemplates = new HashMap<>(); + Map fileModuleTemplates = new HashMap<>(); + for (IngestModuleTemplate template : ingestModuleTemplates) { + if (template.isDataSourceIngestModuleTemplate()) { + dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + if (template.isFileIngestModuleTemplate()) { + fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + } + + /** + * Use the mappings and the ingest pipelines configuration to create + * ordered lists of ingest module templates for each ingest pipeline. + */ + IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); + List firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + + /** + * Add any module templates that were not specified in the pipelines + * configuration to an appropriate pipeline - either the first stage + * data source ingest pipeline or the file ingest pipeline. + */ + for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { + firstStageDataSourceModuleTemplates.add(template); + } + for (IngestModuleTemplate template : fileModuleTemplates.values()) { + fileIngestModuleTemplates.add(template); + } + + /** + * Construct the data source ingest pipelines. + */ + this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); + this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); + + /** + * Construct the file ingest pipelines, one per file ingest thread. + */ + try { + int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); + } + } catch (InterruptedException ex) { + /** + * The current thread was interrupted while blocked on a full queue. + * Blocking should actually never happen here, but reset the + * interrupted flag rather than just swallowing the exception. + */ + Thread.currentThread().interrupt(); + } + } + + /** + * Use an ordered list of ingest module factory class names to create an + * ordered output list of ingest module templates for an ingest pipeline. + * The ingest module templates are removed from the input collection as they + * are added to the output collection. + * + * @param ingestModuleTemplates A mapping of ingest module factory class + * names to ingest module templates. + * @param pipelineConfig An ordered list of ingest module factory class + * names representing an ingest pipeline. + * @return + */ + private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { + List templates = new ArrayList<>(); + for (String moduleClassName : pipelineConfig) { + if (ingestModuleTemplates.containsKey(moduleClassName)) { + templates.add(ingestModuleTemplates.remove(moduleClassName)); + } + } + return templates; + } + + /** + * Starts the first stage of the job. + */ + private void startFirstStage() { + this.stage = DataSourceIngestJob.Stages.FIRST; + + /** + * Start one or both of the first stage ingest progress bars. + */ + if (this.hasFirstStageDataSourceIngestPipeline()) { + this.startDataSourceIngestProgressBar(); + } + if (this.hasFileIngestPipeline()) { + this.startFileIngestProgressBar(); + } + + /** + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + } + + /** + * Schedule the first stage tasks. + */ + if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); + } else if (this.hasFirstStageDataSourceIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this); + + /** + * No data source ingest task has been scheduled for this stage, and + * it is possible, if unlikely, that no file ingest tasks were + * actually scheduled since there are files that get filtered out by + * the tasks scheduler. In this special case, an ingest thread will + * never to check for completion of this stage of the job. + */ + this.checkForStageCompleted(); + } + } + + /** + * Starts the second stage of the ingest job. + */ + private void startSecondStage() { + this.stage = DataSourceIngestJob.Stages.SECOND; + this.startDataSourceIngestProgressBar(); + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + } + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } + + /** + * Checks to see if this job has at least one ingest pipeline. + * + * @return True or false. + */ + boolean hasIngestPipeline() { + return this.hasFirstStageDataSourceIngestPipeline() + || this.hasFileIngestPipeline() + || this.hasSecondStageDataSourceIngestPipeline(); + } + + /** + * Checks to see if this job has a first stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasFirstStageDataSourceIngestPipeline() { + return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if this job has a second stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasSecondStageDataSourceIngestPipeline() { + return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if the job has a file level ingest pipeline. + * + * @return True or false. + */ + private boolean hasFileIngestPipeline() { + return (this.fileIngestPipelines.peek().isEmpty() == false); + } + + /** + * Starts up each of the file and data source level ingest modules to + * collect possible errors. + * + * @return A collection of ingest module startup errors, empty on success. + */ + private List startUpIngestPipelines() { + List errors = new ArrayList<>(); + + // Start up the first stage data source ingest pipeline. + errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); + + // Start up the second stage data source ingest pipeline. + errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); + + // Start up the file ingest pipelines (one per file ingest thread). + for (FileIngestPipeline pipeline : this.fileIngestPipelines) { + errors.addAll(pipeline.startUp()); + if (!errors.isEmpty()) { + // If there are start up errors, the ingest job will not proceed + // and the errors will ultimately be reported to the user for + // possible remedy so shut down the pipelines now that an + // attempt has been made to start up the data source ingest + // pipeline and at least one copy of the file ingest pipeline. + // pipeline. There is no need to complete starting up all of the + // file ingest pipeline copies since any additional start up + // errors are likely redundant. + while (!this.fileIngestPipelines.isEmpty()) { + pipeline = this.fileIngestPipelines.poll(); + List shutDownErrors = pipeline.shutDown(); + if (!shutDownErrors.isEmpty()) { + logIngestModuleErrors(shutDownErrors); + } + } + break; + } + } + + logIngestModuleErrors(errors); + return errors; + } + + /** + * Starts the data source level ingest progress bar. + */ + private void startDataSourceIngestProgressBar() { + synchronized (this.dataSourceIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + this.dataSource.getName()); + this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. What remains to be done is to find out whether + // the user wants to cancel only the currently executing + // data source ingest module or the entire ingest job. + DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); + String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); + JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); + if (panel.cancelAllDataSourceIngestModules()) { + DataSourceIngestJob.this.cancel(); + } else { + DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule(); + } + return true; + } + }); + this.dataSourceIngestProgress.start(); + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + + /** + * Starts the file level ingest progress bar. + */ + private void startFileIngestProgressBar() { + synchronized (this.fileIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. + DataSourceIngestJob.this.cancel(); + return true; + } + }); + this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + this.fileIngestProgress.start(); + this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + } + } + + /** + * Checks to see if the ingest tasks for the current stage are completed and + * does a stage transition if they are. + */ + private void checkForStageCompleted() { + synchronized (this.stageCompletionCheckLock) { + if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) { + switch (this.stage) { + case FIRST: + this.finishFirstStage(); + break; + case SECOND: + this.finish(); + break; + } + } + } + } + + /** + * Shuts down the first stage ingest pipelines and progress bars and starts + * the second stage, if appropriate. + */ + private void finishFirstStage() { + // Shut down the file ingest pipelines. Note that no shut down is + // required for the data source ingest pipeline because data source + // ingest modules do not have a shutdown() method. + List errors = new ArrayList<>(); + while (!this.fileIngestPipelines.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelines.poll(); + errors.addAll(pipeline.shutDown()); + } + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + // Finish the first stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + + // Finish the file ingest progress bar, if it hasn't already + // been finished. + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + this.fileIngestProgress.finish(); + this.fileIngestProgress = null; + } + } + + /** + * Start the second stage, if appropriate. + */ + if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } else { + this.finish(); + } + } + + /** + * Shuts down the ingest pipelines and progress bars for this job. + */ + private void finish() { + this.stage = DataSourceIngestJob.Stages.FINALIZATION; + + // Finish the second stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + + this.parentJob.dataSourceJobFinished(this); + } + + /** + * Write ingest module errors to the log. + * + * @param errors The errors. + */ + private void logIngestModuleErrors(List errors) { + for (IngestModuleError error : errors) { + DataSourceIngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS + } + } + + /** + * Gets the currently running data source level ingest module. + * + * @return The currently running module, may be null. + */ + DataSourceIngestPipeline.DataSourceIngestModuleDecorator getCurrentDataSourceIngestModule() { + if (null != this.currentDataSourceIngestPipeline) { + return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); + } else { + return null; + } + } + + /** + * Requests a temporary cancellation of data source level ingest in order to + * stop the currently executing data source ingest module. + */ + private void cancelCurrentDataSourceIngestModule() { + this.currentDataSourceIngestModuleCancelled = true; + } + + /** + * Gets a snapshot of this jobs state and performance. + * + * @return An ingest job statistics object. + */ + IngestJobSnapshot getSnapshot() { + return new IngestJobSnapshot(); + } + + /** + * Stores basic diagnostic statistics for an ingest job. + */ + class IngestJobSnapshot { + + private final long jobId; + private final String dataSource; + private final long startTime; + private final long processedFiles; + private final long estimatedFilesToProcess; + private final long snapShotTime; + private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; + + /** + * Constructs an object to store basic diagnostic statistics for an + * ingest job. + */ + IngestJobSnapshot() { + this.jobId = DataSourceIngestJob.this.id; + this.dataSource = DataSourceIngestJob.this.dataSource.getName(); + this.startTime = DataSourceIngestJob.this.createTime; + synchronized (DataSourceIngestJob.this.fileIngestProgressLock) { + this.processedFiles = DataSourceIngestJob.this.processedFiles; + this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess; + this.snapShotTime = new Date().getTime(); + } + + /** + * Get a snapshot of the tasks currently in progress for this job. + */ + this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); + } + + /** + * Gets the identifier of the ingest job that is the subject of this + * snapshot. + * + * @return The ingest job id. + */ + long getJobId() { + return this.jobId; + } + + /** + * Gets the name of the data source associated with the ingest job that + * is the subject of this snapshot. + * + * @return A data source name string. + */ + String getDataSource() { + return dataSource; + } + + /** + * Gets files per second throughput since the ingest job that is the + * subject of this snapshot started. + * + * @return Files processed per second (approximate). + */ + double getSpeed() { + return (double) processedFiles / ((snapShotTime - startTime) / 1000); + } + + /** + * Gets the time the ingest job was started. + * + * @return The start time as number of milliseconds since January 1, + * 1970, 00:00:00 GMT. + */ + long getStartTime() { + return startTime; + } + + /** + * Gets time these statistics were collected. + * + * @return The statistics collection time as number of milliseconds + * since January 1, 1970, 00:00:00 GMT. + */ + long getSnapshotTime() { + return snapShotTime; + } + + /** + * Gets the number of files processed for the job so far. + * + * @return The number of processed files. + */ + long getFilesProcessed() { + return processedFiles; + } + + /** + * Gets an estimate of the files that still need to be processed for + * this job. + * + * @return The estimate. + */ + long getFilesEstimated() { + return estimatedFilesToProcess; + } + + long getRootQueueSize() { + return this.tasksSnapshot.getRootQueueSize(); + } + + long getDirQueueSize() { + return this.tasksSnapshot.getDirectoryTasksQueueSize(); + } + + long getFileQueueSize() { + return this.tasksSnapshot.getFileQueueSize(); + } + + long getDsQueueSize() { + return this.tasksSnapshot.getDsQueueSize(); + } + + long getRunningListSize() { + return this.tasksSnapshot.getRunningListSize(); + } + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index bde810e21d..3c0a3361da 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -25,9 +25,9 @@ import org.netbeans.api.progress.ProgressHandle; */ public class DataSourceIngestModuleProgress { - private final IngestJob job; + private final DataSourceIngestJob job; - DataSourceIngestModuleProgress(IngestJob job) { + DataSourceIngestModuleProgress(DataSourceIngestJob job) { this.job = job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 54d06023ee..1ffd7852b0 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -19,37 +19,61 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source ingest modules. It starts them, - * shuts them down, and runs them in sequential order. + * A pipeline consisting of a sequence of data source level ingest modules. The + * pipeline starts the modules, runs them in sequential order, and shuts them + * down. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; + private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); + private volatile DataSourceIngestModuleDecorator currentModule; - DataSourceIngestPipeline(IngestJob job, List moduleTemplates) { + /** + * Constructs a pipeline consisting of a sequence of data source level + * ingest modules. The pipeline starts the modules, runs them in sequential + * order, and shuts them down. + * + * @param job The ingest job to which this pipeline belongs. + * @param moduleTemplates The ingest module templates that define the + * pipeline. + */ + DataSourceIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - // Create an ingest module instance from each data source ingest module - // template. + /** + * Create a data source level ingest module instance from each ingest + * module template. + */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); modules.add(module); } - } + } } + /** + * Indicates whether or not there are any modules in this pipeline. + * + * @return True or false. + */ boolean isEmpty() { return modules.isEmpty(); } + /** + * Starts up the ingest module in this pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ List startUp() { List errors = new ArrayList<>(); for (DataSourceIngestModuleDecorator module : modules) { @@ -62,11 +86,20 @@ final class DataSourceIngestPipeline { return errors; } + /** + * Runs a data source through the ingest modules in sequential order. + * + * @param task A data source level ingest task containing a data source to + * be processed. + * @return A list of ingest module errors, possible empty. + */ List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); for (DataSourceIngestModuleDecorator module : modules) { try { + module.setStartTime(); + this.currentModule = module; String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", module.getDisplayName(), dataSource.getName()); @@ -79,40 +112,97 @@ final class DataSourceIngestPipeline { } if (this.job.isCancelled()) { break; - } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { + } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { this.job.currentDataSourceIngestModuleCancellationCompleted(); } } + this.currentModule = null; ingestManager.setIngestTaskProgressCompleted(task); return errors; } - private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + /** + * Gets the currently running module. + */ + DataSourceIngestModuleDecorator getCurrentlyRunningModule() { + return this.currentModule; + } + + /** + * This class decorates a data source level ingest module with a display + * name and a start time. + */ + static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; + private Date startTime; + /** + * Constructs an object that decorates a data source level ingest module + * with a display name and a running time. + * + * @param module The data source level ingest module to be decorated. + * @param displayName + */ DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; + this.startTime = new Date(); } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { - return module.getClass().getCanonicalName(); + return this.module.getClass().getCanonicalName(); } + /** + * Gets a module name suitable for display in a UI. + * + * @return The display name. + */ String getDisplayName() { - return displayName; + return this.displayName; + } + + /** + * Sets the start time to the current time. + */ + void setStartTime() { + this.startTime = new Date(); + } + + /** + * Gets the time the decorated ingest module started processing the data + * source. + * + * @return The start time. + */ + Date getStartTime() { + return this.startTime; } + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); + this.module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - return module.process(dataSource, statusHelper); + this.startTime = new Date(); + return this.module.process(dataSource, statusHelper); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index 4fecebbe84..66fa744682 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.ingest; final class DataSourceIngestTask extends IngestTask { - DataSourceIngestTask(IngestJob job) { + DataSourceIngestTask(DataSourceIngestJob job) { super(job); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 329c717d5c..0be6d355e9 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -29,10 +29,10 @@ import org.sleuthkit.datamodel.AbstractFile; final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; + private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); - FileIngestPipeline(IngestJob job, List moduleTemplates) { + FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; // Create an ingest module instance from each file ingest module diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 6fa02cf4fc..41a4045b12 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -29,7 +29,7 @@ final class FileIngestTask extends IngestTask { private final AbstractFile file; - FileIngestTask(IngestJob job, AbstractFile file) { + FileIngestTask(DataSourceIngestJob job, AbstractFile file) { super(job); this.file = file; } @@ -53,8 +53,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestJob job = getIngestJob(); - IngestJob otherJob = other.getIngestJob(); + DataSourceIngestJob job = getIngestJob(); + DataSourceIngestJob otherJob = other.getIngestJob(); if (job != otherJob && (job == null || !job.equals(otherJob))) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index ff463321a7..d24121221f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -19,490 +19,133 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.logging.Level; -import javax.swing.JOptionPane; -import org.netbeans.api.progress.ProgressHandle; -import org.netbeans.api.progress.ProgressHandleFactory; -import org.openide.util.Cancellable; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; +import java.util.concurrent.atomic.AtomicLong; import org.sleuthkit.datamodel.Content; /** - * Encapsulates a data source to be processed and the settings, ingest module - * pipelines, and progress bars that are used to process it. + * Runs a collection of data sources through a set of ingest modules specified + * via ingest job settings. */ -final class IngestJob { +public final class IngestJob { - private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); - - /** - * These fields define an ingest job: a unique ID supplied by the ingest - * manager, the user's ingest job settings, and the data source to be - * processed. - */ + private static final AtomicLong nextJobId = new AtomicLong(0L); private final long id; - private final IngestJobSettings ingestJobSettings; - private final Content dataSource; + private final Map dataSourceJobs; + private boolean cancelled; /** - * An ingest job runs in stages. - */ - private static enum Stages { - - /** - * Setting up for processing. - */ - INITIALIZATION, - /** - * Running high priority data source level ingest modules and file level - * ingest modules. - */ - FIRST, - /** - * Running lower priority, usually long-running, data source level - * ingest modules. - */ - SECOND, - /** - * Cleaning up. - */ - FINALIZATION - }; - private Stages stage; - private final Object stageCompletionCheckLock; - - /** - * An ingest job has separate data source level ingest module pipelines for - * each processing stage. Longer running, lower priority modules belong in - * the second stage pipeline. - */ - private final Object dataSourceIngestPipelineLock; - private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; - private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; - - /** - * An ingest job has a collection of identical file level ingest module - * pipelines, one for each file level ingest thread in the ingest manager. - */ - private final LinkedBlockingQueue fileIngestPipelines; - - /** - * An ingest job supports cancellation of either the currently running data - * source level ingest module or the entire ingest job. - */ - private volatile boolean currentDataSourceIngestModuleCancelled; - private volatile boolean cancelled; - - /** - * An ingest job uses the task scheduler singleton to create and queue the - * ingest tasks that make up the job. - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - - /** - * These fields are used to report data source level ingest progress for the - * ingest job. - */ - private final Object dataSourceIngestProgressLock; - private ProgressHandle dataSourceIngestProgress; - - /** - * These fields are used to report file level ingest task progress for the - * ingest job. - */ - private final Object fileIngestProgressLock; - private final List filesInProgress; - private long estimatedFilesToProcess; - private long processedFiles; - private ProgressHandle fileIngestProgress; - - /** - * This field is used to record the creation of the ingest job. - */ - private final long createTime; - - /** - * Constructs an ingest job. + * Constructs an ingest job that runs a collection of data sources through a + * set of ingest modules specified via ingest job settings. * - * @param id The identifier assigned to the job. - * @param dataSource The data source to be ingested. - * @param settings The settings for the ingest job. + * @param dataSources The data sources to be ingested. + * @param settings The ingest job settings. */ - IngestJob(long id, Content dataSource, IngestJobSettings settings) { - this.id = id; - this.dataSource = dataSource; - this.ingestJobSettings = settings; - this.dataSourceIngestPipelineLock = new Object(); - this.fileIngestPipelines = new LinkedBlockingQueue<>(); - this.filesInProgress = new ArrayList<>(); - this.dataSourceIngestProgressLock = new Object(); - this.fileIngestProgressLock = new Object(); - this.stage = IngestJob.Stages.INITIALIZATION; - this.stageCompletionCheckLock = new Object(); - this.createTime = new Date().getTime(); + IngestJob(Collection dataSources, IngestJobSettings settings) { + this.id = IngestJob.nextJobId.getAndIncrement(); + this.dataSourceJobs = new HashMap<>(); + for (Content dataSource : dataSources) { + DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings); + this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); + } } - /** - * Gets the identifier assigned to this job. - * - * @return The job identifier. - */ - long getId() { + public long getId() { return this.id; } - /** - * Gets the data source to be ingested by this job. - * - * @return A Content object representing the data source. - */ - Content getDataSource() { - return this.dataSource; - } - - /** - * Gets whether or not unallocated space should be processed as part of this - * job. - * - * @return True or false. - */ - boolean shouldProcessUnallocatedSpace() { - return this.ingestJobSettings.getProcessUnallocatedSpace(); - } - - /** - * Passes the data source for this job through the currently active data - * source level ingest pipeline. - * - * @param task A data source ingest task wrapping the data source. - */ - void process(DataSourceIngestTask task) { - try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - /** - * Run the data source through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - } + synchronized public ProgressSnapshot getProgressSnapshot() { + DataSourceIngestModuleHandle moduleHandle = null; + Date fileAnalysisStartTime = null; + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + DataSourceIngestPipeline.DataSourceIngestModuleDecorator module = dataSourceJob.getCurrentDataSourceIngestModule(); + if (null != module) { + moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - - /** - * Shut down the data source ingest progress bar right away. Data - * source-level processing is finished for this stage. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. - * - * @param task A file ingest task. - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or putting to the file ingest - * pipelines collection. - */ - void process(FileIngestTask task) throws InterruptedException { - try { - if (!this.isCancelled()) { - /** - * Get a file ingest pipeline not currently in use by another - * file ingest thread. - */ - FileIngestPipeline pipeline = this.fileIngestPipelines.take(); - if (!pipeline.isEmpty()) { - /** - * Get the file to process. - */ - AbstractFile file = task.getFile(); - - /** - * Update the file ingest progress bar. - */ - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); - } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); - } - this.filesInProgress.add(file.getName()); - } - - /** - * Run the file through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - /** - * Update the file ingest progress bar again, in case the - * file was being displayed. - */ - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); - } else { - this.fileIngestProgress.progress(""); - } - } - } - } - - /** - * Relinquish the pipeline so it can be reused by another file - * ingest thread. - */ - this.fileIngestPipelines.put(pipeline); - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Adds more files to an ingest job, i.e., extracted or carved files. Not - * currently supported for the second stage of the job. - * - * @param files A list of files to add. - */ - void addFiles(List files) { - /** - * Note: This implementation assumes that this is being called by an an - * ingest module running code on an ingest thread that is holding a - * reference to an ingest task, so no task completion check is done. - */ - if (IngestJob.Stages.FIRST == this.stage) { - for (AbstractFile file : files) { - IngestJob.taskScheduler.scheduleFileIngestTask(this, file); - } - } else { - IngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + // RJCTODO: File analysis time thing } - /** - * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a task completion check would not be necessary. This is a - * bit of defensive programming. - */ - this.checkForStageCompleted(); + return new ProgressSnapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); } - /** - * Updates the display name of the data source level ingest progress bar. - * - * @param displayName The new display name. - */ - void updateDataSourceIngestProgressBarDisplayName(String displayName) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); - } - } + synchronized public void cancelDataSourceIngestModule(DataSourceIngestModuleHandle module) { + DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); + // RJCTODO: check equality, etc. } - /** - * Switches the data source level ingest progress bar to determinate mode. - * This should be called if the total work units to process the data source - * is known. - * - * @param workUnits Total number of work units for the processing of the - * data source. - */ - void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); - } - } + synchronized public void cancel() { + for (DataSourceIngestJob job : this.dataSourceJobs.values()) { + job.cancel(); } - } - - /** - * Switches the data source level ingest progress bar to indeterminate mode. - * This should be called if the total work units to process the data source - * is unknown. - */ - void switchDataSourceIngestProgressBarToIndeterminate() { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with the number of work - * units performed, if in the determinate mode. - * - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); - } - } - } - } - - /** - * Updates the data source level ingest progress with a new task name, where - * the task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - */ - void advanceDataSourceIngestProgressBar(String currentTask) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with a new task name - * and the number of work units performed, if in the determinate mode. The - * task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); - } - } - } - - /** - * Queries whether or not a temporary cancellation of data source level - * ingest in order to stop the currently executing data source level ingest - * module is in effect. - * - * @return True or false. - */ - boolean currentDataSourceIngestModuleIsCancelled() { - return this.currentDataSourceIngestModuleCancelled; - } - - /** - * Rescind a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module. - */ - void currentDataSourceIngestModuleCancellationCompleted() { - this.currentDataSourceIngestModuleCancelled = false; - - /** - * A new progress bar must be created because the cancel button of the - * previously constructed component is disabled by NetBeans when the - * user selects the "OK" button of the cancellation confirmation dialog - * popped up by NetBeans when the progress bar cancel button was - * pressed. - */ - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - this.startDataSourceIngestProgressBar(); - } - } - - /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. - */ - void cancel() { - /** - * Put a cancellation message on data source level ingest progress bar, - * if it is still running. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (dataSourceIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - dataSource.getName()); - dataSourceIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), - "IngestJob.progress.cancelling", - displayName)); - } - } - - /** - * Put a cancellation message on the file level ingest progress bar, if - * it is still running. - */ - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", - displayName)); - } - } - this.cancelled = true; + } - /** - * Tell the task scheduler to cancel all pending tasks, i.e., tasks not - * not being performed by an ingest thread. - */ - IngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); - this.checkForStageCompleted(); + synchronized public boolean isCancelled() { + return this.cancelled; } /** - * Queries whether or not cancellation of ingest i.e., a shutdown of the - * data source level and file level ingest pipelines, has been requested. - * - * @return True or false. + * A thread-safe snapshot of ingest job progress. */ - boolean isCancelled() { - return this.cancelled; + public static final class ProgressSnapshot { + + private final DataSourceIngestModuleHandle dataSourceModule; + private final Date fileAnalysisStartTime; + private final boolean cancelled; + + private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { + this.dataSourceModule = dataSourceModule; + this.fileAnalysisStartTime = fileAnalysisStartTime; + this.cancelled = cancelled; + } + + public DataSourceIngestModuleHandle runningDataSourceIngestModule() { + /** + * It is safe to hand out this reference because the object is + * immutable and the mutable + * DataSourceIngestPipeline.DataSourceIngestModuleDecorator is + * hidden from public access. + */ + return this.dataSourceModule; + } + + public boolean fileAnalysisIsRunning() { + return (null != this.fileAnalysisStartTime); + } + + public Date fileAnalysisStartTime() { + return new Date(this.fileAnalysisStartTime.getTime()); + } + + public boolean isCancelled() { + return this.cancelled; + } + + } + + public static class DataSourceIngestModuleHandle { + + private final long dataSourceIngestJobId; + private final DataSourceIngestPipeline.DataSourceIngestModuleDecorator module; + + private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.DataSourceIngestModuleDecorator module) { + this.dataSourceIngestJobId = dataSourceIngestJobId; + this.module = module; + } + + public String displayName() { + return this.module.getDisplayName(); + } + + public Date startTime() { + // RJCTODO + return new Date(); + } + } /** @@ -510,536 +153,56 @@ final class IngestJob { * * @return A collection of ingest module startup errors, empty on success. */ - List start(List ingestModuleTemplates) { - this.createIngestPipelines(ingestModuleTemplates); - List errors = startUpIngestPipelines(); - if (errors.isEmpty()) { - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - this.startFirstStage(); - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } + List start() { + boolean hasIngestPipeline = false; + List errors = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + errors.addAll(dataSourceJob.start()); + hasIngestPipeline = dataSourceJob.hasIngestPipeline(); } return errors; } - /** - * Creates the file and data source ingest pipelines. - * - * @param ingestModuleTemplates Ingest module templates to use to populate - * the pipelines. - */ - private void createIngestPipelines(List ingestModuleTemplates) { - /** - * Make mappings of ingest module factory class names to templates. - */ - Map dataSourceModuleTemplates = new HashMap<>(); - Map fileModuleTemplates = new HashMap<>(); - for (IngestModuleTemplate template : ingestModuleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } - if (template.isFileIngestModuleTemplate()) { - fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } - } - - /** - * Use the mappings and the ingest pipelines configuration to create - * ordered lists of ingest module templates for each ingest pipeline. - */ - IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); - - /** - * Add any module templates that were not specified in the pipelines - * configuration to an appropriate pipeline - either the first stage - * data source ingest pipeline or the file ingest pipeline. - */ - for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { - firstStageDataSourceModuleTemplates.add(template); - } - for (IngestModuleTemplate template : fileModuleTemplates.values()) { - fileIngestModuleTemplates.add(template); - } - - /** - * Construct the data source ingest pipelines. - */ - this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - - /** - * Construct the file ingest pipelines, one per file ingest thread. - */ - try { - int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); - } - } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full queue. - * Blocking should actually never happen here, but reset the - * interrupted flag rather than just swallowing the exception. - */ - Thread.currentThread().interrupt(); - } - } - - /** - * Use an ordered list of ingest module factory class names to create an - * ordered output list of ingest module templates for an ingest pipeline. - * The ingest module templates are removed from the input collection as they - * are added to the output collection. - * - * @param ingestModuleTemplates A mapping of ingest module factory class - * names to ingest module templates. - * @param pipelineConfig An ordered list of ingest module factory class - * names representing an ingest pipeline. - * @return - */ - private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { - List templates = new ArrayList<>(); - for (String moduleClassName : pipelineConfig) { - if (ingestModuleTemplates.containsKey(moduleClassName)) { - templates.add(ingestModuleTemplates.remove(moduleClassName)); - } - } - return templates; - } - - /** - * Starts the first stage of the job. - */ - private void startFirstStage() { - this.stage = IngestJob.Stages.FIRST; - - /** - * Start one or both of the first stage ingest progress bars. - */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - IngestJob.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - IngestJob.taskScheduler.scheduleFileIngestTasks(this); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never to check for completion of this stage of the job. - */ - this.checkForStageCompleted(); - } - } - - /** - * Starts the second stage of the ingest job. - */ - private void startSecondStage() { - this.stage = IngestJob.Stages.SECOND; - this.startDataSourceIngestProgressBar(); - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; - } - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } - /** * Checks to see if this job has at least one ingest pipeline. * * @return True or false. */ boolean hasIngestPipeline() { - return this.hasFirstStageDataSourceIngestPipeline() - || this.hasFileIngestPipeline() - || this.hasSecondStageDataSourceIngestPipeline(); - } - - /** - * Checks to see if this job has a first stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasFirstStageDataSourceIngestPipeline() { - return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if this job has a second stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasSecondStageDataSourceIngestPipeline() { - return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if the job has a file level ingest pipeline. - * - * @return True or false. - */ - private boolean hasFileIngestPipeline() { - return (this.fileIngestPipelines.peek().isEmpty() == false); - } - - /** - * Starts up each of the file and data source level ingest modules to - * collect possible errors. - * - * @return A collection of ingest module startup errors, empty on success. - */ - private List startUpIngestPipelines() { - List errors = new ArrayList<>(); - - // Start up the first stage data source ingest pipeline. - errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); - - // Start up the second stage data source ingest pipeline. - errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); - - // Start up the file ingest pipelines (one per file ingest thread). - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { - errors.addAll(pipeline.startUp()); - if (!errors.isEmpty()) { - // If there are start up errors, the ingest job will not proceed - // and the errors will ultimately be reported to the user for - // possible remedy so shut down the pipelines now that an - // attempt has been made to start up the data source ingest - // pipeline and at least one copy of the file ingest pipeline. - // pipeline. There is no need to complete starting up all of the - // file ingest pipeline copies since any additional start up - // errors are likely redundant. - while (!this.fileIngestPipelines.isEmpty()) { - pipeline = this.fileIngestPipelines.poll(); - List shutDownErrors = pipeline.shutDown(); - if (!shutDownErrors.isEmpty()) { - logIngestModuleErrors(shutDownErrors); - } - } + boolean hasIngestPipeline = false; + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + if (dataSourceJob.hasIngestPipeline()) { + hasIngestPipeline = true; break; } } - - logIngestModuleErrors(errors); - return errors; + return hasIngestPipeline; } /** - * Starts the data source level ingest progress bar. - */ - private void startDataSourceIngestProgressBar() { - synchronized (this.dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. What remains to be done is to find out whether - // the user wants to cancel only the currently executing - // data source ingest module or the entire ingest job. - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { - IngestJob.this.cancel(); - } else { - IngestJob.this.cancelCurrentDataSourceIngestModule(); - } - return true; - } - }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - - /** - * Starts the file level ingest progress bar. - */ - private void startFileIngestProgressBar() { - synchronized (this.fileIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. - IngestJob.this.cancel(); - return true; - } - }); - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); - } - } - - /** - * Checks to see if the ingest tasks for the current stage are completed and - * does a stage transition if they are. - */ - private void checkForStageCompleted() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - switch (this.stage) { - case FIRST: - this.finishFirstStage(); - break; - case SECOND: - this.finish(); - break; - } - } - } - } - - /** - * Shuts down the first stage ingest pipelines and progress bars and starts - * the second stage, if appropriate. - */ - private void finishFirstStage() { - // Shut down the file ingest pipelines. Note that no shut down is - // required for the data source ingest pipeline because data source - // ingest modules do not have a shutdown() method. - List errors = new ArrayList<>(); - while (!this.fileIngestPipelines.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelines.poll(); - errors.addAll(pipeline.shutDown()); - } - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; - } - } - - /** - * Start the second stage, if appropriate. - */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } else { - this.finish(); - } - } - - /** - * Shuts down the ingest pipelines and progress bars for this job. - */ - private void finish() { - this.stage = IngestJob.Stages.FINALIZATION; - - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - IngestManager.getInstance().finishJob(this); - } - - /** - * Write ingest module errors to the log. + * Provides a callback for completed data source ingest jobs, allowing the + * ingest job to notify the ingest manager when it is complete. * - * @param errors The errors. + * @param dataSourceIngestJob A completed data source ingest job. */ - private void logIngestModuleErrors(List errors) { - for (IngestModuleError error : errors) { - IngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS + synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { + this.dataSourceJobs.remove(dataSourceIngestJob.getId()); + if (this.dataSourceJobs.isEmpty()) { + IngestManager.getInstance().finishJob(this); } } - + /** - * Requests a temporary cancellation of data source level ingest in order to - * stop the currently executing data source ingest module. - */ - private void cancelCurrentDataSourceIngestModule() { - this.currentDataSourceIngestModuleCancelled = true; - } - - /** - * Gets a snapshot of this jobs state and performance. + * Gets a snapshot of this job's state and performance. * - * @return An ingest job statistics object. + * @return A list of */ - IngestJobSnapshot getSnapshot() { - return new IngestJobSnapshot(); + List getSnapshot() { + List snapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + snapshots.add(dataSourceJob.getSnapshot()); + } + return snapshots; } - - /** - * Stores basic diagnostic statistics for an ingest job. - */ - class IngestJobSnapshot { - - private final long jobId; - private final String dataSource; - private final long startTime; - private final long processedFiles; - private final long estimatedFilesToProcess; - private final long snapShotTime; - private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; - - /** - * Constructs an object to store basic diagnostic statistics for an - * ingest job. - */ - IngestJobSnapshot() { - this.jobId = IngestJob.this.id; - this.dataSource = IngestJob.this.dataSource.getName(); - this.startTime = IngestJob.this.createTime; - synchronized (IngestJob.this.fileIngestProgressLock) { - this.processedFiles = IngestJob.this.processedFiles; - this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; - this.snapShotTime = new Date().getTime(); - } - - /** - * Get a snapshot of the tasks currently in progress for this job. - */ - this.tasksSnapshot = IngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); - } - - /** - * Gets the identifier of the ingest job that is the subject of this - * snapshot. - * - * @return The ingest job id. - */ - long getJobId() { - return this.jobId; - } - - /** - * Gets the name of the data source associated with the ingest job that - * is the subject of this snapshot. - * - * @return A data source name string. - */ - String getDataSource() { - return dataSource; - } - - /** - * Gets files per second throughput since the ingest job that is the - * subject of this snapshot started. - * - * @return Files processed per second (approximate). - */ - double getSpeed() { - return (double) processedFiles / ((snapShotTime - startTime) / 1000); - } - - /** - * Gets the time the ingest job was started. - * - * @return The start time as number of milliseconds since January 1, - * 1970, 00:00:00 GMT. - */ - long getStartTime() { - return startTime; - } - - /** - * Gets time these statistics were collected. - * - * @return The statistics collection time as number of milliseconds - * since January 1, 1970, 00:00:00 GMT. - */ - long getSnapshotTime() { - return snapShotTime; - } - - /** - * Gets the number of files processed for the job so far. - * - * @return The number of processed files. - */ - long getFilesProcessed() { - return processedFiles; - } - - /** - * Gets an estimate of the files that still need to be processed for - * this job. - * - * @return The estimate. - */ - long getFilesEstimated() { - return estimatedFilesToProcess; - } - - long getRootQueueSize() { - return this.tasksSnapshot.getRootQueueSize(); - } - - long getDirQueueSize() { - return this.tasksSnapshot.getDirectoryTasksQueueSize(); - } - - long getFileQueueSize() { - return this.tasksSnapshot.getFileQueueSize(); - } - - long getDsQueueSize() { - return this.tasksSnapshot.getDsQueueSize(); - } - - long getRunningListSize() { - return this.tasksSnapshot.getRunningListSize(); - } - - } - + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 6587a20d19..020c05812b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -28,9 +28,9 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJobContext { - private final IngestJob ingestJob; + private final DataSourceIngestJob ingestJob; - IngestJobContext(IngestJob ingestJob) { + IngestJobContext(DataSourceIngestJob ingestJob) { this.ingestJob = ingestJob; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 7d32adbcd3..624a9fa2ac 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -56,10 +56,8 @@ public class IngestManager { private static IngestManager instance = null; /** - * The ingest manager assigns a unique ID to each ingest job and maintains a - * mapping of job IDs to jobs. + * The ingest manager maintains a mapping of ingest job IDs to ingest jobs. */ - private final AtomicLong nextJobId = new AtomicLong(0L); private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); /** @@ -67,8 +65,8 @@ public class IngestManager { * pools is given a unique thread/task ID. */ // TODO: It is no longer necessary to have multiple thread pools. - private final AtomicLong nextThreadId = new AtomicLong(0L); - + private final AtomicLong nextThreadId = new AtomicLong(0L); + /** * Ingest jobs are started on a pool thread by ingest job starters. A * mapping of thread/task IDs to the result objects associated with each @@ -216,13 +214,18 @@ public class IngestManager { /** * Starts an ingest job, i.e., processing by ingest modules, for each data - * source in a collection of data sources. + * source in a collection of data sources. Note that if the provide UI + * argument is set to true, it is assumed this method is being called on the + * EDT and a worker thread will be dispatched to start the job. * * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param doMessageBoxes Whether or not to display message boxes for errors. + * @param provideUI Whether or not to support user interaction, e.g., + * showing message boxes and reporting progress through the NetBeans + * Progress API. + * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doMessageBoxes) { + public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean provideUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -232,7 +235,7 @@ public class IngestManager { } long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, doMessageBoxes)); + Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, provideUI)); ingestJobStarters.put(taskId, task); } @@ -406,19 +409,19 @@ public class IngestManager { } /** - * Starts an ingest job for a data source. + * Starts an ingest job for a collection of data sources. * - * @param dataSource The data source to ingest. + * @param dataSource The data sources to ingest. * @param settings The settings for the job. * @return A collection of ingest module start up errors, empty on success. */ - private List startJob(Content dataSource, IngestJobSettings settings) { + private List startJob(Collection dataSources, IngestJobSettings settings) { List errors = new ArrayList<>(); if (this.jobCreationIsEnabled) { - long jobId = this.nextJobId.incrementAndGet(); - IngestJob job = new IngestJob(jobId, dataSource, settings); + IngestJob job = new IngestJob(dataSources, settings); + long jobId = job.getId(); this.jobsById.put(jobId, job); - errors = job.start(settings.getEnabledIngestModuleTemplates()); + errors = job.start(); if (errors.isEmpty() && job.hasIngestPipeline()) { this.fireIngestJobStarted(jobId); IngestManager.logger.log(Level.INFO, "Ingest job {0} started", jobId); @@ -628,10 +631,10 @@ public class IngestManager { * * @return A list of ingest job state snapshots. */ - List getIngestJobSnapshots() { - List snapShots = new ArrayList<>(); + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); for (IngestJob job : this.jobsById.values()) { - snapShots.add(job.getSnapshot()); + snapShots.addAll(job.getSnapshot()); } return snapShots; } @@ -684,61 +687,35 @@ public class IngestManager { return true; } }); - progress.start(dataSources.size()); + progress.start(); /** - * Try to start the ingest jobs. + * Try to start the ingest job. */ - int workUnitsCompleted = 0; - for (Content dataSource : this.dataSources) { - - /** - * Cancellation check. - */ - if (Thread.currentThread().isInterrupted()) { - return null; + List errors = IngestManager.this.startJob(this.dataSources, this.settings); + if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); } - - /** - * Add a "subtitle" to the display name of the progress bar - * to indicate an ingest job is being started for this data - * source. - */ - String progressMessage = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.progressingDisplayName", - dataSource.getName()); - progress.progress(progressMessage); - - /** - * Start an ingest job for this data source. - */ - List errors = IngestManager.this.startJob(dataSource, this.settings); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); - } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - } - - progress.progress(progressMessage, ++workUnitsCompleted); + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); } return null; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 47042c8335..7514a597c0 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -163,7 +163,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final String[] columnNames = {"Job ID", "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; - private List jobSnapshots; + private List jobSnapshots; private IngestJobTableModel() { refresh(); @@ -191,7 +191,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { - IngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); + DataSourceIngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); Object cellValue; switch (columnIndex) { case 0: diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 7ff6634f64..cfef33be7a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -23,15 +23,15 @@ import org.sleuthkit.datamodel.Content; abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; - private final IngestJob job; + private final DataSourceIngestJob job; private long threadId; - IngestTask(IngestJob job) { + IngestTask(DataSourceIngestJob job) { this.job = job; threadId = NOT_SET; } - IngestJob getIngestJob() { + DataSourceIngestJob getIngestJob() { return job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index d401b1439c..2adfb3ca40 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -148,7 +148,7 @@ final class IngestTasksScheduler { * @throws InterruptedException if the calling thread is blocked due to a * full tasks queue and is interrupted. */ - synchronized void scheduleIngestTasks(IngestJob job) { + synchronized void scheduleIngestTasks(DataSourceIngestJob job) { // Scheduling of both a data source ingest task and file ingest tasks // for a job must be an atomic operation. Otherwise, the data source // task might be completed before the file tasks are scheduled, @@ -163,7 +163,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleDataSourceIngestTask(IngestJob job) { + synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) { DataSourceIngestTask task = new DataSourceIngestTask(job); this.tasksInProgress.add(task); try { @@ -183,7 +183,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleFileIngestTasks(IngestJob job) { + synchronized void scheduleFileIngestTasks(DataSourceIngestJob job) { // Get the top level files for the data source associated with this job // and add them to the root directories priority queue. List topLevelFiles = getTopLevelFiles(job.getDataSource()); @@ -203,7 +203,7 @@ final class IngestTasksScheduler { * @param job The job for which the tasks are to be scheduled. * @param file The file to be associated with the task. */ - synchronized void scheduleFileIngestTask(IngestJob job, AbstractFile file) { + synchronized void scheduleFileIngestTask(DataSourceIngestJob job, AbstractFile file) { FileIngestTask task = new FileIngestTask(job, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { this.tasksInProgress.add(task); @@ -228,7 +228,7 @@ final class IngestTasksScheduler { * @param job The job for which the query is to be performed. * @return True or false. */ - synchronized boolean tasksForJobAreCompleted(IngestJob job) { + synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) { for (IngestTask task : tasksInProgress) { if (task.getIngestJob().getId() == job.getId()) { return false; @@ -245,7 +245,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to to canceled. */ - synchronized void cancelPendingTasksForIngestJob(IngestJob job) { + synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { long jobId = job.getId(); this.removeTasksForJob(this.rootDirectoryTasks, jobId); this.removeTasksForJob(this.directoryTasks, jobId); From 3cf575029567f49d9c3e7af5d2efb81dea27b900 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 12 Dec 2014 18:52:38 -0500 Subject: [PATCH 3/7] Continue refactoring and implementation of new INgestJob class --- .../autopsy/ingest/DataSourceIngestJob.java | 17 +++- .../ingest/DataSourceIngestPipeline.java | 24 ++--- .../autopsy/ingest/FileIngestPipeline.java | 83 ++++++++++++++--- .../sleuthkit/autopsy/ingest/IngestJob.java | 90 +++++++++++++------ .../autopsy/ingest/IngestManager.java | 6 +- .../ingest/IngestProgressSnapshotPanel.java | 4 +- 6 files changed, 167 insertions(+), 57 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 254510bb6c..6e1b921757 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -328,6 +328,15 @@ final class DataSourceIngestJob { this.checkForStageCompleted(); } + /** + * Queries whether or not this job is running file ingest. + * + * @return True or false. + */ + boolean fileIngestIsRunning() { + return ((DataSourceIngestJob.Stages.FIRST == this.stage) && this.hasFileIngestPipeline()); + } + /** * Updates the display name of the data source level ingest progress bar. * @@ -932,14 +941,14 @@ final class DataSourceIngestJob { * * @return An ingest job statistics object. */ - IngestJobSnapshot getSnapshot() { - return new IngestJobSnapshot(); + Snapshot getSnapshot() { + return new Snapshot(); } /** * Stores basic diagnostic statistics for an ingest job. */ - class IngestJobSnapshot { + class Snapshot { private final long jobId; private final String dataSource; @@ -953,7 +962,7 @@ final class DataSourceIngestJob { * Constructs an object to store basic diagnostic statistics for an * ingest job. */ - IngestJobSnapshot() { + Snapshot() { this.jobId = DataSourceIngestJob.this.id; this.dataSource = DataSourceIngestJob.this.dataSource.getName(); this.startTime = DataSourceIngestJob.this.createTime; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 1ffd7852b0..734effc789 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -25,9 +25,9 @@ import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * A pipeline consisting of a sequence of data source level ingest modules. The - * pipeline starts the modules, runs them in sequential order, and shuts them - * down. + * This class manages a sequence of data source level ingest modules. It starts + * the modules, runs data sources through them, and shuts them down when data + * source level ingest is complete. */ final class DataSourceIngestPipeline { @@ -37,9 +37,9 @@ final class DataSourceIngestPipeline { private volatile DataSourceIngestModuleDecorator currentModule; /** - * Constructs a pipeline consisting of a sequence of data source level - * ingest modules. The pipeline starts the modules, runs them in sequential - * order, and shuts them down. + * Constructs an object that manages a sequence of data source level ingest + * modules. It starts the modules, runs data sources through them, and shuts + * them down when data source level ingest is complete. * * @param job The ingest job to which this pipeline belongs. * @param moduleTemplates The ingest module templates that define the @@ -127,7 +127,7 @@ final class DataSourceIngestPipeline { DataSourceIngestModuleDecorator getCurrentlyRunningModule() { return this.currentModule; } - + /** * This class decorates a data source level ingest module with a display * name and a start time. @@ -143,7 +143,7 @@ final class DataSourceIngestPipeline { * with a display name and a running time. * * @param module The data source level ingest module to be decorated. - * @param displayName + * @param displayName The display name. */ DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { this.module = module; @@ -168,14 +168,14 @@ final class DataSourceIngestPipeline { String getDisplayName() { return this.displayName; } - + /** * Sets the start time to the current time. */ void setStartTime() { this.startTime = new Date(); } - + /** * Gets the time the decorated ingest module started processing the data * source. @@ -202,7 +202,7 @@ final class DataSourceIngestPipeline { this.startTime = new Date(); return this.module.process(dataSource, statusHelper); } - + } - + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 0be6d355e9..3bbdd7fa12 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -19,24 +19,38 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * This class manages a sequence of file ingest modules. It starts them, - * shuts them down, and runs a file through them. + * This class manages a sequence of file level ingest modules. It starts the + * modules, runs files through them, and shuts them down when file level ingest + * is complete. */ final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); + private Date startTime; + /** + * Constructs an object that manages a sequence of file level ingest + * modules. It starts the modules, runs files through them, and shuts them + * down when file level ingest is complete. + * + * @param job The ingest job of which this pipeline is a part. + * @param moduleTemplates The ingest module templates that define the + * pipeline. + */ FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - // Create an ingest module instance from each file ingest module - // template. + /** + * Create an ingest module instance from each file ingest module + * template. + */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); @@ -45,12 +59,28 @@ final class FileIngestPipeline { } } + /** + * Returns the time when the pipeline began processing files. + * + * @return The file processing start time, may be null. + */ + Date getProcessingStartTime() { + return this.startTime; + } + + /** + * Queries whether or not the pipeline has been configured with at least one + * file level ingest module. + * + * @return True or false. + */ boolean isEmpty() { return this.modules.isEmpty(); } /** - * Start up all of the modules in the pipeline. + * Start up all of the modules in the pipeline. + * * @return List of errors or empty list if no errors */ List startUp() { @@ -66,13 +96,15 @@ final class FileIngestPipeline { } /** - * Process the file down the pipeline of modules. - * Startup must have been called before this is called. - * - * @param file File to analyze - * @return List of errors or empty list if no errors + * Runs a file through the ingest modules in sequential order. + * + * @param task A file level ingest task containing a file to be processed. + * @return A list of ingest module errors, possible empty. */ List process(FileIngestTask task) { + if (null == this.startTime) { + this.startTime = new Date(); + } List errors = new ArrayList<>(); AbstractFile file = task.getFile(); for (FileIngestModuleDecorator module : this.modules) { @@ -106,37 +138,68 @@ final class FileIngestPipeline { return errors; } + /** + * This class decorates a file level ingest module with a display name. + */ private static final class FileIngestModuleDecorator implements FileIngestModule { private final FileIngestModule module; private final String displayName; + /** + * Constructs an object that decorates a file level ingest module with a + * display name. + * + * @param module The file level ingest module to be decorated. + * @param displayName The display name. + */ FileIngestModuleDecorator(FileIngestModule module, String displayName) { this.module = module; this.displayName = displayName; } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { return module.getClass().getCanonicalName(); } + /** + * Gets a module name suitable for display in a UI. + * + * @return The display name. + */ String getDisplayName() { return displayName; } + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(AbstractFile file) { return module.process(file); } + /** + * @inheritDoc + */ @Override public void shutDown() { module.shutDown(); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index d24121221f..c7731aecb5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -33,7 +33,7 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJob { - private static final AtomicLong nextJobId = new AtomicLong(0L); + private static final AtomicLong nextId = new AtomicLong(0L); private final long id; private final Map dataSourceJobs; private boolean cancelled; @@ -46,7 +46,7 @@ public final class IngestJob { * @param settings The ingest job settings. */ IngestJob(Collection dataSources, IngestJobSettings settings) { - this.id = IngestJob.nextJobId.getAndIncrement(); + this.id = IngestJob.nextId.getAndIncrement(); this.dataSourceJobs = new HashMap<>(); for (Content dataSource : dataSources) { DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings); @@ -54,36 +54,82 @@ public final class IngestJob { } } + /** + * Gets the unique identifier assigned to this ingest job. + * + * @return The job identifier. + */ public long getId() { return this.id; } - synchronized public ProgressSnapshot getProgressSnapshot() { + /** + * Gets a snapshot of the state and performance of this ingest job. + * + * @return The snapshot. + */ + synchronized public Snapshot getSnapshot() { + // TODO: There are race conditions here that can be easily fixed by + // eliminating the child jobs per plan. DataSourceIngestModuleHandle moduleHandle = null; Date fileAnalysisStartTime = null; for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + /** + * Only one data source ingest module should be running at a time, + * so grab the first one. There is a race condition here. + */ DataSourceIngestPipeline.DataSourceIngestModuleDecorator module = dataSourceJob.getCurrentDataSourceIngestModule(); if (null != module) { moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - // RJCTODO: File analysis time thing + + } - return new ProgressSnapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); + return new Snapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); } - synchronized public void cancelDataSourceIngestModule(DataSourceIngestModuleHandle module) { + /** + * Gets snapshots of the state and performance of this ingest job's child + * data source ingest jobs. + * + * @return A list of data source ingest job snapshots. + */ + synchronized List getDetailedSnapshot() { + List snapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + snapshots.add(dataSourceJob.getSnapshot()); + } + return snapshots; + } + + /** + * + * @param module + */ + synchronized public void cancelIngestModule(DataSourceIngestModuleHandle module) { DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); // RJCTODO: check equality, etc. } + /** + * Requests cancellation of this ingest job, i.e., a shutdown of its data + * source level and file level ingest pipelines. + */ synchronized public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { job.cancel(); } - this.cancelled = true; + this.cancelled = true; // RJCTODO: make children push cancellation up } + /** + * Queries whether or not cancellation of this ingest job, i.e., a shutdown + * of its data source level and file level ingest pipelines, has been + * requested. + * + * @return True or false. + */ synchronized public boolean isCancelled() { return this.cancelled; } @@ -91,13 +137,13 @@ public final class IngestJob { /** * A thread-safe snapshot of ingest job progress. */ - public static final class ProgressSnapshot { + public static final class Snapshot { private final DataSourceIngestModuleHandle dataSourceModule; private final Date fileAnalysisStartTime; private final boolean cancelled; - private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { + private Snapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { this.dataSourceModule = dataSourceModule; this.fileAnalysisStartTime = fileAnalysisStartTime; this.cancelled = cancelled; @@ -127,6 +173,11 @@ public final class IngestJob { } + /** + * A handle to a data source level ingest module that can be used to get + * basic information about the module and to request cancellation, i.e., + * shut down, of the module. + */ public static class DataSourceIngestModuleHandle { private final long dataSourceIngestJobId; @@ -149,9 +200,9 @@ public final class IngestJob { } /** - * Starts up the ingest pipelines and ingest progress bars. + * Starts up the ingest pipelines and ingest progress bars for this job. * - * @return A collection of ingest module startup errors, empty on success. + * @return A collection of ingest module start up errors, empty on success. */ List start() { boolean hasIngestPipeline = false; @@ -164,7 +215,7 @@ public final class IngestJob { } /** - * Checks to see if this job has at least one ingest pipeline. + * Checks to see if this ingest job has at least one ingest pipeline. * * @return True or false. */ @@ -191,18 +242,5 @@ public final class IngestJob { IngestManager.getInstance().finishJob(this); } } - - /** - * Gets a snapshot of this job's state and performance. - * - * @return A list of - */ - List getSnapshot() { - List snapshots = new ArrayList<>(); - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - snapshots.add(dataSourceJob.getSnapshot()); - } - return snapshots; - } - + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 624a9fa2ac..0602579188 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -631,10 +631,10 @@ public class IngestManager { * * @return A list of ingest job state snapshots. */ - List getIngestJobSnapshots() { - List snapShots = new ArrayList<>(); + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); for (IngestJob job : this.jobsById.values()) { - snapShots.addAll(job.getSnapshot()); + snapShots.addAll(job.getDetailedSnapshot()); } return snapShots; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 7514a597c0..b21b1dd3df 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -163,7 +163,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final String[] columnNames = {"Job ID", "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; - private List jobSnapshots; + private List jobSnapshots; private IngestJobTableModel() { refresh(); @@ -191,7 +191,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { - DataSourceIngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); + DataSourceIngestJob.Snapshot snapShot = jobSnapshots.get(rowIndex); Object cellValue; switch (columnIndex) { case 0: From 6c6f5687126840103dd3fad0155f05d28f31dcfa Mon Sep 17 00:00:00 2001 From: Karl Mortensen Date: Mon, 15 Dec 2014 09:04:40 -0500 Subject: [PATCH 4/7] Use final local instead of member variable, add instanceof for JDialog. --- .../autopsy/ingest/IngestProgressSnapshotDialog.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java index 0e59b3ceb5..90ee2b6078 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java @@ -37,8 +37,7 @@ public final class IngestProgressSnapshotDialog extends JDialog { private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestProgressSnapshotDialog.title.text"); private static final Dimension DIMENSIONS = new Dimension(500, 300); - private JDialog pseudoOwner = null; - + /** * Constructs a non-modal instance of the dialog with its own frame. */ @@ -54,8 +53,8 @@ public final class IngestProgressSnapshotDialog extends JDialog { */ public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) { super((Window) owner, TITLE, ModalityType.MODELESS); - if (shouldBeModal) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. - pseudoOwner = (JDialog) owner; + if (shouldBeModal && owner instanceof JDialog) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. + final JDialog pseudoOwner = (JDialog) owner; addWindowListener(new WindowAdapter() { @Override public void windowClosed(WindowEvent e) { // Put it back to how it was before we manipulated it. From 2de22db810f85fd56b6ac4351cd4cfb80841c695 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Dec 2014 16:09:55 -0500 Subject: [PATCH 5/7] Check in new public IngestJob and IngestManager API stubs --- .../autopsy/ingest/DataSourceIngestJob.java | 2 +- .../ingest/DataSourceIngestPipeline.java | 18 +-- .../autopsy/ingest/FileIngestPipeline.java | 77 +++++++--- .../sleuthkit/autopsy/ingest/IngestJob.java | 131 +++++++++++++----- .../autopsy/ingest/IngestManager.java | 35 +++-- 5 files changed, 179 insertions(+), 84 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 6e1b921757..c244a16f7c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -920,7 +920,7 @@ final class DataSourceIngestJob { * * @return The currently running module, may be null. */ - DataSourceIngestPipeline.DataSourceIngestModuleDecorator getCurrentDataSourceIngestModule() { + DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { if (null != this.currentDataSourceIngestPipeline) { return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); } else { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 734effc789..d4dd78ca34 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -28,13 +28,15 @@ import org.sleuthkit.datamodel.Content; * This class manages a sequence of data source level ingest modules. It starts * the modules, runs data sources through them, and shuts them down when data * source level ingest is complete. + *

+ * This class is not thread-safe. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; - private final List modules = new ArrayList<>(); - private volatile DataSourceIngestModuleDecorator currentModule; + private final List modules = new ArrayList<>(); + private volatile PipelineModule currentModule; /** * Constructs an object that manages a sequence of data source level ingest @@ -54,7 +56,7 @@ final class DataSourceIngestPipeline { */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); modules.add(module); } } @@ -76,7 +78,7 @@ final class DataSourceIngestPipeline { */ List startUp() { List errors = new ArrayList<>(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall @@ -96,7 +98,7 @@ final class DataSourceIngestPipeline { List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.setStartTime(); this.currentModule = module; @@ -124,7 +126,7 @@ final class DataSourceIngestPipeline { /** * Gets the currently running module. */ - DataSourceIngestModuleDecorator getCurrentlyRunningModule() { + PipelineModule getCurrentlyRunningModule() { return this.currentModule; } @@ -132,7 +134,7 @@ final class DataSourceIngestPipeline { * This class decorates a data source level ingest module with a display * name and a start time. */ - static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + static class PipelineModule implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; @@ -145,7 +147,7 @@ final class DataSourceIngestPipeline { * @param module The data source level ingest module to be decorated. * @param displayName The display name. */ - DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { + PipelineModule(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; this.startTime = new Date(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 3bbdd7fa12..3e5b09f3ba 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -26,14 +26,17 @@ import org.sleuthkit.datamodel.AbstractFile; /** * This class manages a sequence of file level ingest modules. It starts the * modules, runs files through them, and shuts them down when file level ingest - * is complete. + * is complete. + *

+ * This class is not thread-safe. */ final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; - private final List modules = new ArrayList<>(); + private final List modules = new ArrayList<>(); private Date startTime; + private boolean running; /** * Constructs an object that manages a sequence of file level ingest @@ -53,24 +56,15 @@ final class FileIngestPipeline { */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { - FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); modules.add(module); } } } /** - * Returns the time when the pipeline began processing files. - * - * @return The file processing start time, may be null. - */ - Date getProcessingStartTime() { - return this.startTime; - } - - /** - * Queries whether or not the pipeline has been configured with at least one - * file level ingest module. + * Queries whether or not this pipeline has been configured with at least + * one file level ingest module. * * @return True or false. */ @@ -79,35 +73,54 @@ final class FileIngestPipeline { } /** - * Start up all of the modules in the pipeline. + * Starts up all of the modules in the pipeline. * - * @return List of errors or empty list if no errors + * @return List of start up errors, possibly empty. */ List startUp() { List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + if (this.running) { + throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS + } + + for (PipelineModule module : this.modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } + this.running = true; return errors; } + /** + * Returns the start up time of the pipeline. + * + * @return The file processing start time, may be null. + */ + Date getStartTime() { + return this.startTime; + } + /** * Runs a file through the ingest modules in sequential order. * * @param task A file level ingest task containing a file to be processed. - * @return A list of ingest module errors, possible empty. + * @return A list of processing errors, possible empty. */ List process(FileIngestTask task) { + if (!this.running) { + throw new IllegalStateException("Attempt to process a file with pipeline that is not running"); //NON-NLS + } + if (null == this.startTime) { this.startTime = new Date(); } + List errors = new ArrayList<>(); AbstractFile file = task.getFile(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(file); @@ -126,9 +139,18 @@ final class FileIngestPipeline { return errors; } + /** + * Shuts down all of the modules in the pipeline. + * + * @return A list of shut down errors, possibly empty. + */ List shutDown() { + if (!this.running) { + throw new IllegalStateException("Attempt to shut down a pipeline that is not running"); //NON-NLS + } + List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { module.shutDown(); } catch (Throwable ex) { // Catch-all exception firewall @@ -138,10 +160,19 @@ final class FileIngestPipeline { return errors; } + /** + * Queries whether or not this file ingest level pipeline is running. + * + * @return True or false. + */ + boolean isRunning() { + return this.running; + } + /** * This class decorates a file level ingest module with a display name. */ - private static final class FileIngestModuleDecorator implements FileIngestModule { + private static final class PipelineModule implements FileIngestModule { private final FileIngestModule module; private final String displayName; @@ -153,7 +184,7 @@ final class FileIngestPipeline { * @param module The file level ingest module to be decorated. * @param displayName The display name. */ - FileIngestModuleDecorator(FileIngestModule module, String displayName) { + PipelineModule(FileIngestModule module, String displayName) { this.module = module; this.displayName = displayName; } @@ -168,7 +199,7 @@ final class FileIngestPipeline { } /** - * Gets a module name suitable for display in a UI. + * Gets display name of the decorated ingest module. * * @return The display name. */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index c7731aecb5..9c51e744d9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -68,32 +68,34 @@ public final class IngestJob { * * @return The snapshot. */ - synchronized public Snapshot getSnapshot() { - // TODO: There are race conditions here that can be easily fixed by - // eliminating the child jobs per plan. + synchronized public ProgressSnapshot getSnapshot() { + /** + * There are race conditions in the code that follows, but they are not + * important because this is just a coarse-grained status report. If + * stale data is returned in any single snapshot, it will be corrected + * in subsequent snapshots. + */ DataSourceIngestModuleHandle moduleHandle = null; - Date fileAnalysisStartTime = null; + boolean fileIngestRunning = false; + Date fileIngestStartTime = null; for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - /** - * Only one data source ingest module should be running at a time, - * so grab the first one. There is a race condition here. - */ - DataSourceIngestPipeline.DataSourceIngestModuleDecorator module = dataSourceJob.getCurrentDataSourceIngestModule(); + DataSourceIngestPipeline.PipelineModule module = dataSourceJob.getCurrentDataSourceIngestModule(); if (null != module) { moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - - + + // RJCTODO: For each data source job, check for a running flag and + // get the oldest start data for the start dates, if any. } - return new Snapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); + return new ProgressSnapshot(moduleHandle, fileIngestRunning, fileIngestStartTime, this.cancelled); } /** * Gets snapshots of the state and performance of this ingest job's child * data source ingest jobs. * - * @return A list of data source ingest job snapshots. + * @return A list of data source ingest job progress snapshots. */ synchronized List getDetailedSnapshot() { List snapshots = new ArrayList<>(); @@ -104,29 +106,34 @@ public final class IngestJob { } /** + * Requests cancellation of a specific data source level ingest module. + * Returns immediately, but there may be a delay before the ingest module + * responds by stopping processing, if it is still running when the request + * is made. * - * @param module + * @param module The handle of the data source ingest module to be canceled, + * which can obtained from a progress snapshot. */ synchronized public void cancelIngestModule(DataSourceIngestModuleHandle module) { DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); - // RJCTODO: check equality, etc. + // RJCTODO: Pass through the cancellation request. } /** - * Requests cancellation of this ingest job, i.e., a shutdown of its data - * source level and file level ingest pipelines. + * Requests cancellation of the data source level and file level ingest + * modules of this ingest job. Returns immediately, but there may be a delay + * before all of the ingest modules respond by stopping processing. */ synchronized public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { job.cancel(); } - this.cancelled = true; // RJCTODO: make children push cancellation up + this.cancelled = true; } /** - * Queries whether or not cancellation of this ingest job, i.e., a shutdown - * of its data source level and file level ingest pipelines, has been - * requested. + * Queries whether or not cancellation of the data source level and file + * level ingest modules of this ingest job has been requested. * * @return True or false. */ @@ -135,38 +142,72 @@ public final class IngestJob { } /** - * A thread-safe snapshot of ingest job progress. + * A snapshot of ingest job progress. */ - public static final class Snapshot { + public static final class ProgressSnapshot { private final DataSourceIngestModuleHandle dataSourceModule; - private final Date fileAnalysisStartTime; + private final boolean fileIngestRunning; + private final Date fileIngestStartTime; private final boolean cancelled; - private Snapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { + /** + * Constructs a snapshot of ingest job progress. + * + * @param dataSourceModule The currently running data source level + * ingest module, may be null + * @param fileIngestRunning Whether or not file ingest is currently + * running. + * @param fileIngestStartTime The start time of file level ingest, may + * be null + * @param cancelled Whether or not a cancellation request has been + * issued. + */ + private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) { this.dataSourceModule = dataSourceModule; - this.fileAnalysisStartTime = fileAnalysisStartTime; + this.fileIngestRunning = fileIngestRunning; + this.fileIngestStartTime = fileIngestStartTime; this.cancelled = cancelled; } + /** + * Gets a handle to the currently running data source level ingest + * module at the time the snapshot is taken. + * + * @return The handle, may be null. + */ public DataSourceIngestModuleHandle runningDataSourceIngestModule() { /** * It is safe to hand out this reference because the object is - * immutable and the mutable - * DataSourceIngestPipeline.DataSourceIngestModuleDecorator is - * hidden from public access. + * immutable. */ return this.dataSourceModule; } - public boolean fileAnalysisIsRunning() { - return (null != this.fileAnalysisStartTime); + /** + * Queries whether or not file level ingest is running at the time the + * snapshot is taken. + * + * @return True or false. + */ + public boolean fileIngestIsRunning() { + return this.fileIngestRunning; } - public Date fileAnalysisStartTime() { - return new Date(this.fileAnalysisStartTime.getTime()); + /** + * Gets the time that file level ingest started. + * + * @return The start time, may be null. + */ + public Date fileIngestStartTime() { + return new Date(this.fileIngestStartTime.getTime()); } + /** + * Queries whether or not a cancellation request has been issued. + * + * @return True or false. + */ public boolean isCancelled() { return this.cancelled; } @@ -181,20 +222,36 @@ public final class IngestJob { public static class DataSourceIngestModuleHandle { private final long dataSourceIngestJobId; - private final DataSourceIngestPipeline.DataSourceIngestModuleDecorator module; + private final DataSourceIngestPipeline.PipelineModule module; - private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.DataSourceIngestModuleDecorator module) { + /** + * Constructs a handle to a data source level ingest module that can be + * used to get basic information about the module and to request + * cancellation of the module. + */ + private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.PipelineModule module) { this.dataSourceIngestJobId = dataSourceIngestJobId; this.module = module; } + /** + * Gets the display name of the data source level ingest module + * associated with this handle. + * + * @return The display name. + */ public String displayName() { return this.module.getDisplayName(); } + /** + * Returns the time the data source level ingest module associated with + * this handle began processing. + * + * @return The module start time. + */ public Date startTime() { - // RJCTODO - return new Date(); + return this.module.getStartTime(); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 0602579188..13d4f4211e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -213,19 +213,15 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for each data - * source in a collection of data sources. Note that if the provide UI - * argument is set to true, it is assumed this method is being called on the - * EDT and a worker thread will be dispatched to start the job. + * Starts an ingest job for a collection of data sources. * * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param provideUI Whether or not to support user interaction, e.g., - * showing message boxes and reporting progress through the NetBeans - * Progress API. + * @param doUI Whether or not to support user interaction, e.g., showing + * message boxes and reporting progress through the NetBeans Progress API. * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean provideUI) { + public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -234,9 +230,17 @@ public class IngestManager { ingestMonitor.start(); } - long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, provideUI)); - ingestJobStarters.put(taskId, task); + if (doUI) { + /** + * Assume request is from code running on the EDT and dispatch to a + * worker thread. + */ + long taskId = nextThreadId.incrementAndGet(); + Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, dataSources, settings, doUI)); + ingestJobStarters.put(taskId, task); + } else { + this.startJob(dataSources, settings); + } } /** @@ -245,6 +249,8 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { + // RJCTODO: This may return the wrong answer if an IngestJobStarter has + // been dispatched to the startIngestJobsThreadPool. return !this.jobsById.isEmpty(); } @@ -640,10 +646,9 @@ public class IngestManager { } /** - * Creates and starts an ingest job, i.e., processing by ingest modules, for - * each data source in a collection of data sources. + * Creates and starts an ingest job for a collection of data sources. */ - private final class IngestJobsStarter implements Callable { + private final class IngestJobStarter implements Callable { private final long threadId; private final Collection dataSources; @@ -651,7 +656,7 @@ public class IngestManager { private final boolean doStartupErrorsMsgBox; private ProgressHandle progress; - IngestJobsStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { + IngestJobStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { this.threadId = threadId; this.dataSources = dataSources; this.settings = settings; From 644a4e8c3881ab6f91cbd07e745921badce2bda8 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Dec 2014 16:14:21 -0500 Subject: [PATCH 6/7] Check in new public IngestJob and IngestManager API stubs --- Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java | 2 +- .../core/core.jar/org/netbeans/core/startup/Bundle.properties | 2 +- .../org/netbeans/core/windows/view/ui/Bundle.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 13d4f4211e..3b791dbfce 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -221,7 +221,7 @@ public class IngestManager { * message boxes and reporting progress through the NetBeans Progress API. * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doUI) { + public synchronized void startIngestJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index b808dd0d96..575a34a2f6 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 17:10:10 -0400 +#Mon, 15 Dec 2014 16:14:04 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index f0109bc92b..9ca23c2a58 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 12:13:36 -0400 +#Mon, 15 Dec 2014 16:14:04 -0500 CTL_MainWindow_Title=Autopsy 3.1.1 CTL_MainWindow_Title_No_Project=Autopsy 3.1.1 From 389e014b2ba6aa4dc35fd590d51c4dce5761f0ca Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Dec 2014 16:17:41 -0500 Subject: [PATCH 7/7] Check in new public IngestJob and IngestManager API stubs --- .../autopsy/casemodule/AddImageWizardIngestConfigPanel.java | 2 +- .../org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java | 4 ++-- .../org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 96208f007f..052d46047c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,7 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel dataSources) { - IngestManager.getInstance().startIngestJobs(dataSources, this.settings, true); + IngestManager.getInstance().startIngestJob(dataSources, this.settings, true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index d8a7536ac1..b69e967ad9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -199,7 +199,7 @@ public final class RunIngestModulesDialog extends JDialog { ingestJobSettings.save(); showWarnings(ingestJobSettings); if (startIngestJob) { - IngestManager.getInstance().startIngestJobs(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); + IngestManager.getInstance().startIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); } setVisible(false); dispose();