diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 96208f007f..052d46047c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,7 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import javax.swing.JOptionPane; +import org.netbeans.api.progress.ProgressHandle; +import org.netbeans.api.progress.ProgressHandleFactory; +import org.openide.util.Cancellable; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; + +/** + * Encapsulates a data source to be processed and the settings, ingest module + * pipelines, and progress bars that are used to process it. + */ +final class DataSourceIngestJob { + + private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName()); + private static final AtomicLong nextJobId = new AtomicLong(0L); + + /** + * These fields define a data source ingest job: the parent ingest job, a + * unique ID, the user's ingest job settings, and the data source to be + * processed. + */ + private final IngestJob parentJob; + private final long id; + private final IngestJobSettings settings; + private final Content dataSource; + + /** + * An ingest job runs in stages. + */ + private static enum Stages { + + /** + * Setting up for processing. + */ + INITIALIZATION, + /** + * Running high priority data source level ingest modules and file level + * ingest modules. + */ + FIRST, + /** + * Running lower priority, usually long-running, data source level + * ingest modules. + */ + SECOND, + /** + * Cleaning up. + */ + FINALIZATION + }; + private Stages stage; + private final Object stageCompletionCheckLock; + + /** + * An ingest job has separate data source level ingest module pipelines for + * each processing stage. Longer running, lower priority modules belong in + * the second stage pipeline. + */ + private final Object dataSourceIngestPipelineLock; + private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; + private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; + private DataSourceIngestPipeline currentDataSourceIngestPipeline; + + /** + * An ingest job has a collection of identical file level ingest module + * pipelines, one for each file level ingest thread in the ingest manager. + */ + private final LinkedBlockingQueue fileIngestPipelines; + + /** + * An ingest job supports cancellation of either the currently running data + * source level ingest module or the entire ingest job. + */ + private volatile boolean currentDataSourceIngestModuleCancelled; + private volatile boolean cancelled; + + /** + * An ingest job uses the task scheduler singleton to create and queue the + * ingest tasks that make up the job. + */ + private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + + /** + * These fields are used to report data source level ingest progress for the + * ingest job. + */ + private final Object dataSourceIngestProgressLock; + private ProgressHandle dataSourceIngestProgress; + + /** + * These fields are used to report file level ingest task progress for the + * ingest job. + */ + private final Object fileIngestProgressLock; + private final List filesInProgress; + private long estimatedFilesToProcess; + private long processedFiles; + private ProgressHandle fileIngestProgress; + + /** + * This field is used to record the creation of the ingest job. + */ + private final long createTime; + + /** + * Constructs an ingest job. + * + * @param parentJob The ingest job of which this data source ingest job is a + * part. + * @param dataSource The data source to be ingested. + * @param settings The settings for the ingest job. + */ + DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings) { + this.parentJob = parentJob; + this.id = DataSourceIngestJob.nextJobId.getAndIncrement(); + this.dataSource = dataSource; + this.settings = settings; + this.dataSourceIngestPipelineLock = new Object(); + this.fileIngestPipelines = new LinkedBlockingQueue<>(); + this.filesInProgress = new ArrayList<>(); + this.dataSourceIngestProgressLock = new Object(); + this.fileIngestProgressLock = new Object(); + this.stage = DataSourceIngestJob.Stages.INITIALIZATION; + this.stageCompletionCheckLock = new Object(); + this.createTime = new Date().getTime(); + } + + /** + * Gets the identifier assigned to this job. + * + * @return The job identifier. + */ + long getId() { + return this.id; + } + + /** + * Gets the data source to be ingested by this job. + * + * @return A Content object representing the data source. + */ + Content getDataSource() { + return this.dataSource; + } + + /** + * Gets whether or not unallocated space should be processed as part of this + * job. + * + * @return True or false. + */ + boolean shouldProcessUnallocatedSpace() { + return this.settings.getProcessUnallocatedSpace(); + } + + /** + * Passes the data source for this job through the currently active data + * source level ingest pipeline. + * + * @param task A data source ingest task wrapping the data source. + */ + void process(DataSourceIngestTask task) { + try { + synchronized (this.dataSourceIngestPipelineLock) { + if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + /** + * Run the data source through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } + + /** + * Shut down the data source ingest progress bar right away. Data + * source-level processing is finished for this stage. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Passes a file from the data source for this job through the file level + * ingest pipeline. + * + * @param task A file ingest task. + * @throws InterruptedException if the thread executing this code is + * interrupted while blocked on taking from or putting to the file ingest + * pipelines collection. + */ + void process(FileIngestTask task) throws InterruptedException { + try { + if (!this.isCancelled()) { + /** + * Get a file ingest pipeline not currently in use by another + * file ingest thread. + */ + FileIngestPipeline pipeline = this.fileIngestPipelines.take(); + if (!pipeline.isEmpty()) { + /** + * Get the file to process. + */ + AbstractFile file = task.getFile(); + + /** + * Update the file ingest progress bar. + */ + synchronized (this.fileIngestProgressLock) { + ++this.processedFiles; + if (this.processedFiles <= this.estimatedFilesToProcess) { + this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); + } else { + this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); + } + this.filesInProgress.add(file.getName()); + } + + /** + * Run the file through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(pipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + /** + * Update the file ingest progress bar again, in case the + * file was being displayed. + */ + if (!this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.filesInProgress.remove(file.getName()); + if (this.filesInProgress.size() > 0) { + this.fileIngestProgress.progress(this.filesInProgress.get(0)); + } else { + this.fileIngestProgress.progress(""); + } + } + } + } + + /** + * Relinquish the pipeline so it can be reused by another file + * ingest thread. + */ + this.fileIngestPipelines.put(pipeline); + } + } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Adds more files to an ingest job, i.e., extracted or carved files. Not + * currently supported for the second stage of the job. + * + * @param files A list of files to add. + */ + void addFiles(List files) { + /** + * Note: This implementation assumes that this is being called by an an + * ingest module running code on an ingest thread that is holding a + * reference to an ingest task, so no task completion check is done. + */ + if (DataSourceIngestJob.Stages.FIRST == this.stage) { + for (AbstractFile file : files) { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); + } + } else { + DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + } + + /** + * The intended clients of this method are ingest modules running code + * on an ingest thread that is holding a reference to an ingest task, in + * which case a task completion check would not be necessary. This is a + * bit of defensive programming. + */ + this.checkForStageCompleted(); + } + + /** + * Queries whether or not this job is running file ingest. + * + * @return True or false. + */ + boolean fileIngestIsRunning() { + return ((DataSourceIngestJob.Stages.FIRST == this.stage) && this.hasFileIngestPipeline()); + } + + /** + * Updates the display name of the data source level ingest progress bar. + * + * @param displayName The new display name. + */ + void updateDataSourceIngestProgressBarDisplayName(String displayName) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.setDisplayName(displayName); + } + } + } + + /** + * Switches the data source level ingest progress bar to determinate mode. + * This should be called if the total work units to process the data source + * is known. + * + * @param workUnits Total number of work units for the processing of the + * data source. + */ + void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToDeterminate(workUnits); + } + } + } + } + + /** + * Switches the data source level ingest progress bar to indeterminate mode. + * This should be called if the total work units to process the data source + * is unknown. + */ + void switchDataSourceIngestProgressBarToIndeterminate() { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + } + } + + /** + * Updates the data source level ingest progress bar with the number of work + * units performed, if in the determinate mode. + * + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(int workUnits) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress("", workUnits); + } + } + } + } + + /** + * Updates the data source level ingest progress with a new task name, where + * the task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + */ + void advanceDataSourceIngestProgressBar(String currentTask) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress(currentTask); + } + } + } + } + + /** + * Updates the data source level ingest progress bar with a new task name + * and the number of work units performed, if in the determinate mode. The + * task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { + if (!this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.dataSourceIngestProgress.progress(currentTask, workUnits); + } + } + } + + /** + * Queries whether or not a temporary cancellation of data source level + * ingest in order to stop the currently executing data source level ingest + * module is in effect. + * + * @return True or false. + */ + boolean currentDataSourceIngestModuleIsCancelled() { + return this.currentDataSourceIngestModuleCancelled; + } + + /** + * Rescind a temporary cancellation of data source level ingest that was + * used to stop a single data source level ingest module. + */ + void currentDataSourceIngestModuleCancellationCompleted() { + this.currentDataSourceIngestModuleCancelled = false; + + /** + * A new progress bar must be created because the cancel button of the + * previously constructed component is disabled by NetBeans when the + * user selects the "OK" button of the cancellation confirmation dialog + * popped up by NetBeans when the progress bar cancel button was + * pressed. + */ + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + this.startDataSourceIngestProgressBar(); + } + } + + /** + * Requests cancellation of ingest, i.e., a shutdown of the data source + * level and file level ingest pipelines. + */ + void cancel() { + /** + * Put a cancellation message on data source level ingest progress bar, + * if it is still running. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (dataSourceIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + dataSource.getName()); + dataSourceIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), + "IngestJob.progress.cancelling", + displayName)); + } + } + + /** + * Put a cancellation message on the file level ingest progress bar, if + * it is still running. + */ + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", + displayName)); + } + } + + this.cancelled = true; + + /** + * Tell the task scheduler to cancel all pending tasks, i.e., tasks not + * not being performed by an ingest thread. + */ + DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); + this.checkForStageCompleted(); + } + + /** + * Queries whether or not cancellation of ingest i.e., a shutdown of the + * data source level and file level ingest pipelines, has been requested. + * + * @return True or false. + */ + boolean isCancelled() { + return this.cancelled; + } + + /** + * Starts up the ingest pipelines and ingest progress bars. + * + * @return A collection of ingest module startup errors, empty on success. + */ + List start() { + this.createIngestPipelines(settings.getEnabledIngestModuleTemplates()); + List errors = startUpIngestPipelines(); + if (errors.isEmpty()) { + if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { + this.startFirstStage(); + } else if (this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } + } + return errors; + } + + /** + * Creates the file and data source ingest pipelines. + * + * @param ingestModuleTemplates Ingest module templates to use to populate + * the pipelines. + */ + private void createIngestPipelines(List ingestModuleTemplates) { + /** + * Make mappings of ingest module factory class names to templates. + */ + Map dataSourceModuleTemplates = new HashMap<>(); + Map fileModuleTemplates = new HashMap<>(); + for (IngestModuleTemplate template : ingestModuleTemplates) { + if (template.isDataSourceIngestModuleTemplate()) { + dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + if (template.isFileIngestModuleTemplate()) { + fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + } + + /** + * Use the mappings and the ingest pipelines configuration to create + * ordered lists of ingest module templates for each ingest pipeline. + */ + IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); + List firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + + /** + * Add any module templates that were not specified in the pipelines + * configuration to an appropriate pipeline - either the first stage + * data source ingest pipeline or the file ingest pipeline. + */ + for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { + firstStageDataSourceModuleTemplates.add(template); + } + for (IngestModuleTemplate template : fileModuleTemplates.values()) { + fileIngestModuleTemplates.add(template); + } + + /** + * Construct the data source ingest pipelines. + */ + this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); + this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); + + /** + * Construct the file ingest pipelines, one per file ingest thread. + */ + try { + int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); + } + } catch (InterruptedException ex) { + /** + * The current thread was interrupted while blocked on a full queue. + * Blocking should actually never happen here, but reset the + * interrupted flag rather than just swallowing the exception. + */ + Thread.currentThread().interrupt(); + } + } + + /** + * Use an ordered list of ingest module factory class names to create an + * ordered output list of ingest module templates for an ingest pipeline. + * The ingest module templates are removed from the input collection as they + * are added to the output collection. + * + * @param ingestModuleTemplates A mapping of ingest module factory class + * names to ingest module templates. + * @param pipelineConfig An ordered list of ingest module factory class + * names representing an ingest pipeline. + * @return + */ + private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { + List templates = new ArrayList<>(); + for (String moduleClassName : pipelineConfig) { + if (ingestModuleTemplates.containsKey(moduleClassName)) { + templates.add(ingestModuleTemplates.remove(moduleClassName)); + } + } + return templates; + } + + /** + * Starts the first stage of the job. + */ + private void startFirstStage() { + this.stage = DataSourceIngestJob.Stages.FIRST; + + /** + * Start one or both of the first stage ingest progress bars. + */ + if (this.hasFirstStageDataSourceIngestPipeline()) { + this.startDataSourceIngestProgressBar(); + } + if (this.hasFileIngestPipeline()) { + this.startFileIngestProgressBar(); + } + + /** + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + } + + /** + * Schedule the first stage tasks. + */ + if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); + } else if (this.hasFirstStageDataSourceIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this); + + /** + * No data source ingest task has been scheduled for this stage, and + * it is possible, if unlikely, that no file ingest tasks were + * actually scheduled since there are files that get filtered out by + * the tasks scheduler. In this special case, an ingest thread will + * never to check for completion of this stage of the job. + */ + this.checkForStageCompleted(); + } + } + + /** + * Starts the second stage of the ingest job. + */ + private void startSecondStage() { + this.stage = DataSourceIngestJob.Stages.SECOND; + this.startDataSourceIngestProgressBar(); + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + } + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } + + /** + * Checks to see if this job has at least one ingest pipeline. + * + * @return True or false. + */ + boolean hasIngestPipeline() { + return this.hasFirstStageDataSourceIngestPipeline() + || this.hasFileIngestPipeline() + || this.hasSecondStageDataSourceIngestPipeline(); + } + + /** + * Checks to see if this job has a first stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasFirstStageDataSourceIngestPipeline() { + return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if this job has a second stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasSecondStageDataSourceIngestPipeline() { + return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if the job has a file level ingest pipeline. + * + * @return True or false. + */ + private boolean hasFileIngestPipeline() { + return (this.fileIngestPipelines.peek().isEmpty() == false); + } + + /** + * Starts up each of the file and data source level ingest modules to + * collect possible errors. + * + * @return A collection of ingest module startup errors, empty on success. + */ + private List startUpIngestPipelines() { + List errors = new ArrayList<>(); + + // Start up the first stage data source ingest pipeline. + errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); + + // Start up the second stage data source ingest pipeline. + errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); + + // Start up the file ingest pipelines (one per file ingest thread). + for (FileIngestPipeline pipeline : this.fileIngestPipelines) { + errors.addAll(pipeline.startUp()); + if (!errors.isEmpty()) { + // If there are start up errors, the ingest job will not proceed + // and the errors will ultimately be reported to the user for + // possible remedy so shut down the pipelines now that an + // attempt has been made to start up the data source ingest + // pipeline and at least one copy of the file ingest pipeline. + // pipeline. There is no need to complete starting up all of the + // file ingest pipeline copies since any additional start up + // errors are likely redundant. + while (!this.fileIngestPipelines.isEmpty()) { + pipeline = this.fileIngestPipelines.poll(); + List shutDownErrors = pipeline.shutDown(); + if (!shutDownErrors.isEmpty()) { + logIngestModuleErrors(shutDownErrors); + } + } + break; + } + } + + logIngestModuleErrors(errors); + return errors; + } + + /** + * Starts the data source level ingest progress bar. + */ + private void startDataSourceIngestProgressBar() { + synchronized (this.dataSourceIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + this.dataSource.getName()); + this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. What remains to be done is to find out whether + // the user wants to cancel only the currently executing + // data source ingest module or the entire ingest job. + DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); + String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); + JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); + if (panel.cancelAllDataSourceIngestModules()) { + DataSourceIngestJob.this.cancel(); + } else { + DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule(); + } + return true; + } + }); + this.dataSourceIngestProgress.start(); + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + + /** + * Starts the file level ingest progress bar. + */ + private void startFileIngestProgressBar() { + synchronized (this.fileIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. + DataSourceIngestJob.this.cancel(); + return true; + } + }); + this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + this.fileIngestProgress.start(); + this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + } + } + + /** + * Checks to see if the ingest tasks for the current stage are completed and + * does a stage transition if they are. + */ + private void checkForStageCompleted() { + synchronized (this.stageCompletionCheckLock) { + if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) { + switch (this.stage) { + case FIRST: + this.finishFirstStage(); + break; + case SECOND: + this.finish(); + break; + } + } + } + } + + /** + * Shuts down the first stage ingest pipelines and progress bars and starts + * the second stage, if appropriate. + */ + private void finishFirstStage() { + // Shut down the file ingest pipelines. Note that no shut down is + // required for the data source ingest pipeline because data source + // ingest modules do not have a shutdown() method. + List errors = new ArrayList<>(); + while (!this.fileIngestPipelines.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelines.poll(); + errors.addAll(pipeline.shutDown()); + } + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + // Finish the first stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + + // Finish the file ingest progress bar, if it hasn't already + // been finished. + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + this.fileIngestProgress.finish(); + this.fileIngestProgress = null; + } + } + + /** + * Start the second stage, if appropriate. + */ + if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } else { + this.finish(); + } + } + + /** + * Shuts down the ingest pipelines and progress bars for this job. + */ + private void finish() { + this.stage = DataSourceIngestJob.Stages.FINALIZATION; + + // Finish the second stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + + this.parentJob.dataSourceJobFinished(this); + } + + /** + * Write ingest module errors to the log. + * + * @param errors The errors. + */ + private void logIngestModuleErrors(List errors) { + for (IngestModuleError error : errors) { + DataSourceIngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS + } + } + + /** + * Gets the currently running data source level ingest module. + * + * @return The currently running module, may be null. + */ + DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { + if (null != this.currentDataSourceIngestPipeline) { + return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); + } else { + return null; + } + } + + /** + * Requests a temporary cancellation of data source level ingest in order to + * stop the currently executing data source ingest module. + */ + private void cancelCurrentDataSourceIngestModule() { + this.currentDataSourceIngestModuleCancelled = true; + } + + /** + * Gets a snapshot of this jobs state and performance. + * + * @return An ingest job statistics object. + */ + Snapshot getSnapshot() { + return new Snapshot(); + } + + /** + * Stores basic diagnostic statistics for an ingest job. + */ + class Snapshot { + + private final long jobId; + private final String dataSource; + private final long startTime; + private final long processedFiles; + private final long estimatedFilesToProcess; + private final long snapShotTime; + private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; + + /** + * Constructs an object to store basic diagnostic statistics for an + * ingest job. + */ + Snapshot() { + this.jobId = DataSourceIngestJob.this.id; + this.dataSource = DataSourceIngestJob.this.dataSource.getName(); + this.startTime = DataSourceIngestJob.this.createTime; + synchronized (DataSourceIngestJob.this.fileIngestProgressLock) { + this.processedFiles = DataSourceIngestJob.this.processedFiles; + this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess; + this.snapShotTime = new Date().getTime(); + } + + /** + * Get a snapshot of the tasks currently in progress for this job. + */ + this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); + } + + /** + * Gets the identifier of the ingest job that is the subject of this + * snapshot. + * + * @return The ingest job id. + */ + long getJobId() { + return this.jobId; + } + + /** + * Gets the name of the data source associated with the ingest job that + * is the subject of this snapshot. + * + * @return A data source name string. + */ + String getDataSource() { + return dataSource; + } + + /** + * Gets files per second throughput since the ingest job that is the + * subject of this snapshot started. + * + * @return Files processed per second (approximate). + */ + double getSpeed() { + return (double) processedFiles / ((snapShotTime - startTime) / 1000); + } + + /** + * Gets the time the ingest job was started. + * + * @return The start time as number of milliseconds since January 1, + * 1970, 00:00:00 GMT. + */ + long getStartTime() { + return startTime; + } + + /** + * Gets time these statistics were collected. + * + * @return The statistics collection time as number of milliseconds + * since January 1, 1970, 00:00:00 GMT. + */ + long getSnapshotTime() { + return snapShotTime; + } + + /** + * Gets the number of files processed for the job so far. + * + * @return The number of processed files. + */ + long getFilesProcessed() { + return processedFiles; + } + + /** + * Gets an estimate of the files that still need to be processed for + * this job. + * + * @return The estimate. + */ + long getFilesEstimated() { + return estimatedFilesToProcess; + } + + long getRootQueueSize() { + return this.tasksSnapshot.getRootQueueSize(); + } + + long getDirQueueSize() { + return this.tasksSnapshot.getDirectoryTasksQueueSize(); + } + + long getFileQueueSize() { + return this.tasksSnapshot.getFileQueueSize(); + } + + long getDsQueueSize() { + return this.tasksSnapshot.getDsQueueSize(); + } + + long getRunningListSize() { + return this.tasksSnapshot.getRunningListSize(); + } + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index bde810e21d..3c0a3361da 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -25,9 +25,9 @@ import org.netbeans.api.progress.ProgressHandle; */ public class DataSourceIngestModuleProgress { - private final IngestJob job; + private final DataSourceIngestJob job; - DataSourceIngestModuleProgress(IngestJob job) { + DataSourceIngestModuleProgress(DataSourceIngestJob job) { this.job = job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 54d06023ee..d4dd78ca34 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -19,40 +19,66 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source ingest modules. It starts them, - * shuts them down, and runs them in sequential order. + * This class manages a sequence of data source level ingest modules. It starts + * the modules, runs data sources through them, and shuts them down when data + * source level ingest is complete. + *

+ * This class is not thread-safe. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; - private final List modules = new ArrayList<>(); + private final DataSourceIngestJob job; + private final List modules = new ArrayList<>(); + private volatile PipelineModule currentModule; - DataSourceIngestPipeline(IngestJob job, List moduleTemplates) { + /** + * Constructs an object that manages a sequence of data source level ingest + * modules. It starts the modules, runs data sources through them, and shuts + * them down when data source level ingest is complete. + * + * @param job The ingest job to which this pipeline belongs. + * @param moduleTemplates The ingest module templates that define the + * pipeline. + */ + DataSourceIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - // Create an ingest module instance from each data source ingest module - // template. + /** + * Create a data source level ingest module instance from each ingest + * module template. + */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); modules.add(module); } - } + } } + /** + * Indicates whether or not there are any modules in this pipeline. + * + * @return True or false. + */ boolean isEmpty() { return modules.isEmpty(); } + /** + * Starts up the ingest module in this pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ List startUp() { List errors = new ArrayList<>(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall @@ -62,11 +88,20 @@ final class DataSourceIngestPipeline { return errors; } + /** + * Runs a data source through the ingest modules in sequential order. + * + * @param task A data source level ingest task containing a data source to + * be processed. + * @return A list of ingest module errors, possible empty. + */ List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { + module.setStartTime(); + this.currentModule = module; String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", module.getDisplayName(), dataSource.getName()); @@ -79,40 +114,97 @@ final class DataSourceIngestPipeline { } if (this.job.isCancelled()) { break; - } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { + } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { this.job.currentDataSourceIngestModuleCancellationCompleted(); } } + this.currentModule = null; ingestManager.setIngestTaskProgressCompleted(task); return errors; } - private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + /** + * Gets the currently running module. + */ + PipelineModule getCurrentlyRunningModule() { + return this.currentModule; + } + + /** + * This class decorates a data source level ingest module with a display + * name and a start time. + */ + static class PipelineModule implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; + private Date startTime; - DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { + /** + * Constructs an object that decorates a data source level ingest module + * with a display name and a running time. + * + * @param module The data source level ingest module to be decorated. + * @param displayName The display name. + */ + PipelineModule(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; + this.startTime = new Date(); } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { - return module.getClass().getCanonicalName(); + return this.module.getClass().getCanonicalName(); } + /** + * Gets a module name suitable for display in a UI. + * + * @return The display name. + */ String getDisplayName() { - return displayName; + return this.displayName; } + /** + * Sets the start time to the current time. + */ + void setStartTime() { + this.startTime = new Date(); + } + + /** + * Gets the time the decorated ingest module started processing the data + * source. + * + * @return The start time. + */ + Date getStartTime() { + return this.startTime; + } + + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); + this.module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - return module.process(dataSource, statusHelper); + this.startTime = new Date(); + return this.module.process(dataSource, statusHelper); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index 4fecebbe84..66fa744682 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.ingest; final class DataSourceIngestTask extends IngestTask { - DataSourceIngestTask(IngestJob job) { + DataSourceIngestTask(DataSourceIngestJob job) { super(job); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 329c717d5c..3e5b09f3ba 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -19,63 +19,108 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * This class manages a sequence of file ingest modules. It starts them, - * shuts them down, and runs a file through them. + * This class manages a sequence of file level ingest modules. It starts the + * modules, runs files through them, and shuts them down when file level ingest + * is complete. + *

+ * This class is not thread-safe. */ final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; - private final List modules = new ArrayList<>(); + private final DataSourceIngestJob job; + private final List modules = new ArrayList<>(); + private Date startTime; + private boolean running; - FileIngestPipeline(IngestJob job, List moduleTemplates) { + /** + * Constructs an object that manages a sequence of file level ingest + * modules. It starts the modules, runs files through them, and shuts them + * down when file level ingest is complete. + * + * @param job The ingest job of which this pipeline is a part. + * @param moduleTemplates The ingest module templates that define the + * pipeline. + */ + FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - // Create an ingest module instance from each file ingest module - // template. + /** + * Create an ingest module instance from each file ingest module + * template. + */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { - FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); modules.add(module); } } } + /** + * Queries whether or not this pipeline has been configured with at least + * one file level ingest module. + * + * @return True or false. + */ boolean isEmpty() { return this.modules.isEmpty(); } /** - * Start up all of the modules in the pipeline. - * @return List of errors or empty list if no errors + * Starts up all of the modules in the pipeline. + * + * @return List of start up errors, possibly empty. */ List startUp() { List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + if (this.running) { + throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS + } + + for (PipelineModule module : this.modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } + this.running = true; return errors; } /** - * Process the file down the pipeline of modules. - * Startup must have been called before this is called. - * - * @param file File to analyze - * @return List of errors or empty list if no errors + * Returns the start up time of the pipeline. + * + * @return The file processing start time, may be null. + */ + Date getStartTime() { + return this.startTime; + } + + /** + * Runs a file through the ingest modules in sequential order. + * + * @param task A file level ingest task containing a file to be processed. + * @return A list of processing errors, possible empty. */ List process(FileIngestTask task) { + if (!this.running) { + throw new IllegalStateException("Attempt to process a file with pipeline that is not running"); //NON-NLS + } + + if (null == this.startTime) { + this.startTime = new Date(); + } + List errors = new ArrayList<>(); AbstractFile file = task.getFile(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(file); @@ -94,9 +139,18 @@ final class FileIngestPipeline { return errors; } + /** + * Shuts down all of the modules in the pipeline. + * + * @return A list of shut down errors, possibly empty. + */ List shutDown() { + if (!this.running) { + throw new IllegalStateException("Attempt to shut down a pipeline that is not running"); //NON-NLS + } + List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { module.shutDown(); } catch (Throwable ex) { // Catch-all exception firewall @@ -106,37 +160,77 @@ final class FileIngestPipeline { return errors; } - private static final class FileIngestModuleDecorator implements FileIngestModule { + /** + * Queries whether or not this file ingest level pipeline is running. + * + * @return True or false. + */ + boolean isRunning() { + return this.running; + } + + /** + * This class decorates a file level ingest module with a display name. + */ + private static final class PipelineModule implements FileIngestModule { private final FileIngestModule module; private final String displayName; - FileIngestModuleDecorator(FileIngestModule module, String displayName) { + /** + * Constructs an object that decorates a file level ingest module with a + * display name. + * + * @param module The file level ingest module to be decorated. + * @param displayName The display name. + */ + PipelineModule(FileIngestModule module, String displayName) { this.module = module; this.displayName = displayName; } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { return module.getClass().getCanonicalName(); } + /** + * Gets display name of the decorated ingest module. + * + * @return The display name. + */ String getDisplayName() { return displayName; } + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(AbstractFile file) { return module.process(file); } + /** + * @inheritDoc + */ @Override public void shutDown() { module.shutDown(); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 6fa02cf4fc..41a4045b12 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -29,7 +29,7 @@ final class FileIngestTask extends IngestTask { private final AbstractFile file; - FileIngestTask(IngestJob job, AbstractFile file) { + FileIngestTask(DataSourceIngestJob job, AbstractFile file) { super(job); this.file = file; } @@ -53,8 +53,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestJob job = getIngestJob(); - IngestJob otherJob = other.getIngestJob(); + DataSourceIngestJob job = getIngestJob(); + DataSourceIngestJob otherJob = other.getIngestJob(); if (job != otherJob && (job == null || !job.equals(otherJob))) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 98dbbb3930..9c51e744d9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -19,1029 +19,285 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.logging.Level; -import javax.swing.JOptionPane; -import org.netbeans.api.progress.ProgressHandle; -import org.netbeans.api.progress.ProgressHandleFactory; -import org.openide.util.Cancellable; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; +import java.util.concurrent.atomic.AtomicLong; import org.sleuthkit.datamodel.Content; /** - * Encapsulates a data source to be processed and the settings, ingest module - * pipelines, and progress bars that are used to process it. + * Runs a collection of data sources through a set of ingest modules specified + * via ingest job settings. */ -final class IngestJob { +public final class IngestJob { - private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); - - /** - * These fields define the ingest job: a unique ID supplied by the ingest - * manager, the user's ingest job settings, and the data source to be - * processed. - */ + private static final AtomicLong nextId = new AtomicLong(0L); private final long id; - private final IngestJobSettings ingestJobSettings; - private final Content dataSource; + private final Map dataSourceJobs; + private boolean cancelled; /** - * An ingest job runs in stages. - */ - private static enum Stages { - - /** - * Setting up for processing. - */ - INITIALIZATION, - /** - * Running high priority data source level ingest modules and file level - * ingest modules. - */ - FIRST, - /** - * Running lower priority, usually long-running, data source level - * ingest modules. - */ - SECOND, - /** - * Cleaning up. - */ - FINALIZATION - }; - private Stages stage; - private final Object stageCompletionCheckLock; - - /** - * There is a data source level ingest modules pipeline for both the first - * stage and the second stage. Longer running, lower priority data source - * level ingest modules belong in the second stage pipeline. - */ - private final Object dataSourceIngestPipelineLock; - private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; - private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; - - /** - * There is a collection of identical file level ingest module pipelines, - * one for each file level ingest thread in the ingest manager. - */ - private final LinkedBlockingQueue fileIngestPipelines; - - /** - * The task scheduler singleton is responsible for creating and scheduling - * the ingest tasks that make up this ingest jobs. - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - - /** - * These fields are used to provide data source level task progress bars for - * the job. - */ - private final Object dataSourceIngestProgressLock; - private ProgressHandle dataSourceIngestProgress; - - /** - * These fields are used to provide file level ingest task progress bars for - * the job. - */ - private final Object fileIngestProgressLock; - private final List filesInProgress; - private long estimatedFilesToProcess; - private long processedFiles; - private ProgressHandle fileIngestProgress; - - /** - * These fields support cancellation of either the currently running data - * source level ingest module or the entire ingest job. - */ - private volatile boolean currentDataSourceIngestModuleCancelled; - private volatile boolean cancelled; - - /** - * This field is used for generating ingest job diagnostic data. - */ - private final long startTime; - - /** - * Constructs an ingest job. + * Constructs an ingest job that runs a collection of data sources through a + * set of ingest modules specified via ingest job settings. * - * @param id The identifier assigned to the job. - * @param dataSource The data source to be ingested. - * @param processUnallocatedSpace Whether or not unallocated space should be - * processed during the ingest job. + * @param dataSources The data sources to be ingested. + * @param settings The ingest job settings. */ - IngestJob(long id, Content dataSource, IngestJobSettings settings) { - this.id = id; - this.dataSource = dataSource; - this.ingestJobSettings = settings; - this.dataSourceIngestPipelineLock = new Object(); - this.fileIngestPipelines = new LinkedBlockingQueue<>(); - this.filesInProgress = new ArrayList<>(); - this.dataSourceIngestProgressLock = new Object(); - this.fileIngestProgressLock = new Object(); - this.stage = IngestJob.Stages.INITIALIZATION; - this.stageCompletionCheckLock = new Object(); - this.startTime = new Date().getTime(); + IngestJob(Collection dataSources, IngestJobSettings settings) { + this.id = IngestJob.nextId.getAndIncrement(); + this.dataSourceJobs = new HashMap<>(); + for (Content dataSource : dataSources) { + DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings); + this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); + } } /** - * Gets the identifier assigned to this job. + * Gets the unique identifier assigned to this ingest job. * * @return The job identifier. */ - long getId() { + public long getId() { return this.id; } /** - * Gets the data source to be ingested by this job. + * Gets a snapshot of the state and performance of this ingest job. * - * @return A Content object representing the data source. + * @return The snapshot. */ - Content getDataSource() { - return this.dataSource; - } - - /** - * Gets whether or not unallocated space should be processed as part of this - * job. - * - * @return True or false. - */ - boolean shouldProcessUnallocatedSpace() { - return this.ingestJobSettings.getProcessUnallocatedSpace(); - } - - /** - * Passes the data source for this job through the currently active data - * source level ingest pipeline. - * - * @param task A data source ingest task wrapping the data source. - */ - void process(DataSourceIngestTask task) { - try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - /** - * Run the data source through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - } - } - - /** - * Shut down the data source ingest progress bar right away. Data - * source-level processing is finished for this stage. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. - * - * @param task A file ingest task. - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or putting to the file ingest - * pipelines collection. - */ - void process(FileIngestTask task) throws InterruptedException { - try { - if (!this.isCancelled()) { - /** - * Get a file ingest pipeline not currently in use by another - * file ingest thread. - */ - FileIngestPipeline pipeline = this.fileIngestPipelines.take(); - if (!pipeline.isEmpty()) { - /** - * Get the file to process. - */ - AbstractFile file = task.getFile(); - - /** - * Update the file ingest progress bar. - */ - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); - } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); - } - this.filesInProgress.add(file.getName()); - } - - /** - * Run the file through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - /** - * Update the file ingest progress bar again, in case the - * file was being displayed. - */ - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); - } else { - this.fileIngestProgress.progress(""); - } - } - } - } - - /** - * Relinquish the pipeline so it can be reused by another file - * ingest thread. - */ - this.fileIngestPipelines.put(pipeline); - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Adds more files to an ingest job, i.e., extracted or carved files. Not - * currently supported for the second stage of the job. - * - * @param files A list of files to add. - */ - void addFiles(List files) { + synchronized public ProgressSnapshot getSnapshot() { /** - * Note: This implementation assumes that this is being called by an an - * ingest module running code on an ingest thread that is holding a - * reference to an ingest task, so no task completion check is done. + * There are race conditions in the code that follows, but they are not + * important because this is just a coarse-grained status report. If + * stale data is returned in any single snapshot, it will be corrected + * in subsequent snapshots. */ - if (IngestJob.Stages.FIRST == this.stage) { - for (AbstractFile file : files) { - IngestJob.taskScheduler.scheduleFileIngestTask(this, file); + DataSourceIngestModuleHandle moduleHandle = null; + boolean fileIngestRunning = false; + Date fileIngestStartTime = null; + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + DataSourceIngestPipeline.PipelineModule module = dataSourceJob.getCurrentDataSourceIngestModule(); + if (null != module) { + moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - } else { - IngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + + // RJCTODO: For each data source job, check for a running flag and + // get the oldest start data for the start dates, if any. } - /** - * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a task completion check would not be necessary. This is a - * bit of defensive programming. - */ - this.checkForStageCompleted(); + return new ProgressSnapshot(moduleHandle, fileIngestRunning, fileIngestStartTime, this.cancelled); } /** - * Updates the display name of the data source level ingest progress bar. + * Gets snapshots of the state and performance of this ingest job's child + * data source ingest jobs. * - * @param displayName The new display name. + * @return A list of data source ingest job progress snapshots. */ - void updateDataSourceIngestProgressBarDisplayName(String displayName) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); - } + synchronized List getDetailedSnapshot() { + List snapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + snapshots.add(dataSourceJob.getSnapshot()); } + return snapshots; } /** - * Switches the data source level ingest progress bar to determinate mode. - * This should be called if the total work units to process the data source - * is known. + * Requests cancellation of a specific data source level ingest module. + * Returns immediately, but there may be a delay before the ingest module + * responds by stopping processing, if it is still running when the request + * is made. * - * @param workUnits Total number of work units for the processing of the - * data source. + * @param module The handle of the data source ingest module to be canceled, + * which can obtained from a progress snapshot. */ - void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); - } - } - } + synchronized public void cancelIngestModule(DataSourceIngestModuleHandle module) { + DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); + // RJCTODO: Pass through the cancellation request. } /** - * Switches the data source level ingest progress bar to indeterminate mode. - * This should be called if the total work units to process the data source - * is unknown. + * Requests cancellation of the data source level and file level ingest + * modules of this ingest job. Returns immediately, but there may be a delay + * before all of the ingest modules respond by stopping processing. */ - void switchDataSourceIngestProgressBarToIndeterminate() { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } + synchronized public void cancel() { + for (DataSourceIngestJob job : this.dataSourceJobs.values()) { + job.cancel(); } - } - - /** - * Updates the data source level ingest progress bar with the number of work - * units performed, if in the determinate mode. - * - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); - } - } - } - } - - /** - * Updates the data source level ingest progress with a new task name, where - * the task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - */ - void advanceDataSourceIngestProgressBar(String currentTask) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with a new task name - * and the number of work units performed, if in the determinate mode. The - * task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); - } - } - } - - /** - * Queries whether or not a temporary cancellation of data source level - * ingest in order to stop the currently executing data source level ingest - * module is in effect. - * - * @return True or false. - */ - boolean currentDataSourceIngestModuleIsCancelled() { - return this.currentDataSourceIngestModuleCancelled; - } - - /** - * Rescind a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module. - */ - void currentDataSourceIngestModuleCancellationCompleted() { - this.currentDataSourceIngestModuleCancelled = false; - - /** - * A new progress bar must be created because the cancel button of the - * previously constructed component is disabled by NetBeans when the - * user selects the "OK" button of the cancellation confirmation dialog - * popped up by NetBeans when the progress bar cancel button was - * pressed. - */ - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - this.startDataSourceIngestProgressBar(); - } - } - - /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. - */ - void cancel() { - /** - * Put a cancellation message on data source level ingest progress bar, - * if it is still running. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (dataSourceIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - dataSource.getName()); - dataSourceIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), - "IngestJob.progress.cancelling", - displayName)); - } - } - - /** - * Put a cancellation message on the file level ingest progress bar, if - * it is still running. - */ - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", - displayName)); - } - } - this.cancelled = true; - - /** - * Tell the task scheduler to cancel all pending tasks, i.e., tasks not - * not being performed by an ingest thread. - */ - IngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); - this.checkForStageCompleted(); } /** - * Queries whether or not cancellation of ingest i.e., a shutdown of the - * data source level and file level ingest pipelines, has been requested. + * Queries whether or not cancellation of the data source level and file + * level ingest modules of this ingest job has been requested. * * @return True or false. */ - boolean isCancelled() { + synchronized public boolean isCancelled() { return this.cancelled; } /** - * Starts up the ingest pipelines and ingest progress bars. - * - * @return A collection of ingest module startup errors, empty on success. + * A snapshot of ingest job progress. */ - List start(List ingestModuleTemplates) { - this.createIngestPipelines(ingestModuleTemplates); - List errors = startUpIngestPipelines(); - if (errors.isEmpty()) { - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - this.startFirstStage(); - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } + public static final class ProgressSnapshot { + + private final DataSourceIngestModuleHandle dataSourceModule; + private final boolean fileIngestRunning; + private final Date fileIngestStartTime; + private final boolean cancelled; + + /** + * Constructs a snapshot of ingest job progress. + * + * @param dataSourceModule The currently running data source level + * ingest module, may be null + * @param fileIngestRunning Whether or not file ingest is currently + * running. + * @param fileIngestStartTime The start time of file level ingest, may + * be null + * @param cancelled Whether or not a cancellation request has been + * issued. + */ + private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) { + this.dataSourceModule = dataSourceModule; + this.fileIngestRunning = fileIngestRunning; + this.fileIngestStartTime = fileIngestStartTime; + this.cancelled = cancelled; + } + + /** + * Gets a handle to the currently running data source level ingest + * module at the time the snapshot is taken. + * + * @return The handle, may be null. + */ + public DataSourceIngestModuleHandle runningDataSourceIngestModule() { + /** + * It is safe to hand out this reference because the object is + * immutable. + */ + return this.dataSourceModule; + } + + /** + * Queries whether or not file level ingest is running at the time the + * snapshot is taken. + * + * @return True or false. + */ + public boolean fileIngestIsRunning() { + return this.fileIngestRunning; + } + + /** + * Gets the time that file level ingest started. + * + * @return The start time, may be null. + */ + public Date fileIngestStartTime() { + return new Date(this.fileIngestStartTime.getTime()); + } + + /** + * Queries whether or not a cancellation request has been issued. + * + * @return True or false. + */ + public boolean isCancelled() { + return this.cancelled; + } + + } + + /** + * A handle to a data source level ingest module that can be used to get + * basic information about the module and to request cancellation, i.e., + * shut down, of the module. + */ + public static class DataSourceIngestModuleHandle { + + private final long dataSourceIngestJobId; + private final DataSourceIngestPipeline.PipelineModule module; + + /** + * Constructs a handle to a data source level ingest module that can be + * used to get basic information about the module and to request + * cancellation of the module. + */ + private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.PipelineModule module) { + this.dataSourceIngestJobId = dataSourceIngestJobId; + this.module = module; + } + + /** + * Gets the display name of the data source level ingest module + * associated with this handle. + * + * @return The display name. + */ + public String displayName() { + return this.module.getDisplayName(); + } + + /** + * Returns the time the data source level ingest module associated with + * this handle began processing. + * + * @return The module start time. + */ + public Date startTime() { + return this.module.getStartTime(); + } + + } + + /** + * Starts up the ingest pipelines and ingest progress bars for this job. + * + * @return A collection of ingest module start up errors, empty on success. + */ + List start() { + boolean hasIngestPipeline = false; + List errors = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + errors.addAll(dataSourceJob.start()); + hasIngestPipeline = dataSourceJob.hasIngestPipeline(); } return errors; } /** - * Creates the file and data source ingest pipelines. - * - * @param ingestModuleTemplates Ingest module templates to use to populate - * the pipelines. - */ - private void createIngestPipelines(List ingestModuleTemplates) { - /** - * Make mappings of ingest module factory class names to templates. - */ - Map dataSourceModuleTemplates = new HashMap<>(); - Map fileModuleTemplates = new HashMap<>(); - for (IngestModuleTemplate template : ingestModuleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } - if (template.isFileIngestModuleTemplate()) { - fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } - } - - /** - * Use the mappings and the ingest pipelines configuration to create - * ordered lists of ingest module templates for each ingest pipeline. - */ - IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); - - /** - * Add any module templates that were not specified in the pipelines - * configuration to an appropriate pipeline - either the first stage - * data source ingest pipeline or the file ingest pipeline. - */ - for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { - firstStageDataSourceModuleTemplates.add(template); - } - for (IngestModuleTemplate template : fileModuleTemplates.values()) { - fileIngestModuleTemplates.add(template); - } - - /** - * Construct the data source ingest pipelines. - */ - this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - - /** - * Construct the file ingest pipelines, one per file ingest thread. - */ - try { - int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); - } - } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full queue. - * Blocking should actually never happen here, but reset the - * interrupted flag rather than just swallowing the exception. - */ - Thread.currentThread().interrupt(); - } - } - - /** - * Use an ordered list of ingest module factory class names to create an - * ordered output list of ingest module templates for an ingest pipeline. - * The ingest module templates are removed from the input collection as they - * are added to the output collection. - * - * @param ingestModuleTemplates A mapping of ingest module factory class - * names to ingest module templates. - * @param pipelineConfig An ordered list of ingest module factory class - * names representing an ingest pipeline. - * @return - */ - private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { - List templates = new ArrayList<>(); - for (String moduleClassName : pipelineConfig) { - if (ingestModuleTemplates.containsKey(moduleClassName)) { - templates.add(ingestModuleTemplates.remove(moduleClassName)); - } - } - return templates; - } - - /** - * Starts the first stage of the job. - */ - private void startFirstStage() { - this.stage = IngestJob.Stages.FIRST; - - /** - * Start one or both of the first stage ingest progress bars. - */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - IngestJob.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - IngestJob.taskScheduler.scheduleFileIngestTasks(this); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never to check for completion of this stage of the job. - */ - this.checkForStageCompleted(); - } - } - - /** - * Starts the second stage of the ingest job. - */ - private void startSecondStage() { - this.stage = IngestJob.Stages.SECOND; - this.startDataSourceIngestProgressBar(); - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; - } - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } - - /** - * Checks to see if this job has at least one ingest pipeline. + * Checks to see if this ingest job has at least one ingest pipeline. * * @return True or false. */ boolean hasIngestPipeline() { - return this.hasFirstStageDataSourceIngestPipeline() - || this.hasFileIngestPipeline() - || this.hasSecondStageDataSourceIngestPipeline(); - } - - /** - * Checks to see if this job has a first stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasFirstStageDataSourceIngestPipeline() { - return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if this job has a second stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasSecondStageDataSourceIngestPipeline() { - return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if the job has a file level ingest pipeline. - * - * @return True or false. - */ - private boolean hasFileIngestPipeline() { - return (this.fileIngestPipelines.peek().isEmpty() == false); - } - - /** - * Starts up each of the file and data source level ingest modules to - * collect possible errors. - * - * @return A collection of ingest module startup errors, empty on success. - */ - private List startUpIngestPipelines() { - List errors = new ArrayList<>(); - - // Start up the first stage data source ingest pipeline. - errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); - - // Start up the second stage data source ingest pipeline. - errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); - - // Start up the file ingest pipelines (one per file ingest thread). - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { - errors.addAll(pipeline.startUp()); - if (!errors.isEmpty()) { - // If there are start up errors, the ingest job will not proceed - // and the errors will ultimately be reported to the user for - // possible remedy so shut down the pipelines now that an - // attempt has been made to start up the data source ingest - // pipeline and at least one copy of the file ingest pipeline. - // pipeline. There is no need to complete starting up all of the - // file ingest pipeline copies since any additional start up - // errors are likely redundant. - while (!this.fileIngestPipelines.isEmpty()) { - pipeline = this.fileIngestPipelines.poll(); - List shutDownErrors = pipeline.shutDown(); - if (!shutDownErrors.isEmpty()) { - logIngestModuleErrors(shutDownErrors); - } - } + boolean hasIngestPipeline = false; + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + if (dataSourceJob.hasIngestPipeline()) { + hasIngestPipeline = true; break; } } - - logIngestModuleErrors(errors); - return errors; + return hasIngestPipeline; } /** - * Starts the data source level ingest progress bar. - */ - private void startDataSourceIngestProgressBar() { - synchronized (this.dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. What remains to be done is to find out whether - // the user wants to cancel only the currently executing - // data source ingest module or the entire ingest job. - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { - IngestJob.this.cancel(); - } else { - IngestJob.this.cancelCurrentDataSourceIngestModule(); - } - return true; - } - }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - - /** - * Starts the file level ingest progress bar. - */ - private void startFileIngestProgressBar() { - synchronized (this.fileIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. - IngestJob.this.cancel(); - return true; - } - }); - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); - } - } - - /** - * Checks to see if the ingest tasks for the current stage are completed and - * does a stage transition if they are. - */ - private void checkForStageCompleted() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - switch (this.stage) { - case FIRST: - this.finishFirstStage(); - break; - case SECOND: - this.finish(); - break; - } - } - } - } - - /** - * Shuts down the first stage ingest pipelines and progress bars and starts - * the second stage, if appropriate. - */ - private void finishFirstStage() { - // Shut down the file ingest pipelines. Note that no shut down is - // required for the data source ingest pipeline because data source - // ingest modules do not have a shutdown() method. - List errors = new ArrayList<>(); - while (!this.fileIngestPipelines.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelines.poll(); - errors.addAll(pipeline.shutDown()); - } - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; - } - } - - /** - * Start the second stage, if appropriate. - */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } else { - this.finish(); - } - } - - /** - * Shuts down the ingest pipelines and progress bars for this job. - */ - private void finish() { - this.stage = IngestJob.Stages.FINALIZATION; - - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - IngestManager.getInstance().finishJob(this); - } - - /** - * Write ingest module errors to the log. + * Provides a callback for completed data source ingest jobs, allowing the + * ingest job to notify the ingest manager when it is complete. * - * @param errors The errors. + * @param dataSourceIngestJob A completed data source ingest job. */ - private void logIngestModuleErrors(List errors) { - for (IngestModuleError error : errors) { - IngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS + synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { + this.dataSourceJobs.remove(dataSourceIngestJob.getId()); + if (this.dataSourceJobs.isEmpty()) { + IngestManager.getInstance().finishJob(this); } } - /** - * Requests a temporary cancellation of data source level ingest in order to - * stop the currently executing data source ingest module. - */ - private void cancelCurrentDataSourceIngestModule() { - this.currentDataSourceIngestModuleCancelled = true; - } - - /** - * Gets a snapshot of this jobs state and performance. - * - * @return An ingest job statistics object. - */ - IngestJobSnapshot getSnapshot() { - return new IngestJobSnapshot(); - } - - /** - * Stores basic diagnostic statistics for an ingest job. - */ - class IngestJobSnapshot { - - private final long jobId; - private final String dataSource; - private final long startTime; - private final long processedFiles; - private final long estimatedFilesToProcess; - private final long snapShotTime; - private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; - - /** - * Constructs an object to stores basic diagnostic statistics for an - * ingest job. - */ - IngestJobSnapshot() { - this.jobId = IngestJob.this.id; - this.dataSource = IngestJob.this.dataSource.getName(); - this.startTime = IngestJob.this.startTime; - synchronized (IngestJob.this.fileIngestProgressLock) { - this.processedFiles = IngestJob.this.processedFiles; - this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; - this.snapShotTime = new Date().getTime(); - } - - /** - * Get a snapshot of the tasks currently in progress for this job. - */ - this.tasksSnapshot = IngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); - } - - /** - * Gets the identifier of the ingest job that is the subject of this - * snapshot. - * - * @return The ingest job id. - */ - long getJobId() { - return this.jobId; - } - - /** - * Gets the name of the data source associated with the ingest job that - * is the subject of this snapshot. - * - * @return A data source name string. - */ - String getDataSource() { - return dataSource; - } - - /** - * Gets files per second throughput since the ingest job that is the - * subject of this snapshot started. - * - * @return Files processed per second (approximate). - */ - double getSpeed() { - return (double) processedFiles / ((snapShotTime - startTime) / 1000); - } - - /** - * Gets the time the ingest job was started. - * - * @return The start time as number of milliseconds since January 1, - * 1970, 00:00:00 GMT. - */ - long getStartTime() { - return startTime; - } - - /** - * Gets time these statistics were collected. - * - * @return The statistics collection time as number of milliseconds - * since January 1, 1970, 00:00:00 GMT. - */ - long getSnapshotTime() { - return snapShotTime; - } - - /** - * Gets the number of files processed for the job so far. - * - * @return The number of processed files. - */ - long getFilesProcessed() { - return processedFiles; - } - - /** - * Gets an estimate of the files that still need to be processed for - * this job. - * - * @return The estimate. - */ - long getFilesEstimated() { - return estimatedFilesToProcess; - } - - long getRootQueueSize() { - return this.tasksSnapshot.getRootQueueSize(); - } - - long getDirQueueSize() { - return this.tasksSnapshot.getDirectoryTasksQueueSize(); - } - - long getFileQueueSize() { - return this.tasksSnapshot.getFileQueueSize(); - } - - long getDsQueueSize() { - return this.tasksSnapshot.getDsQueueSize(); - } - - long getRunningListSize() { - return this.tasksSnapshot.getRunningListSize(); - } - - } - } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java index 5c646aeaa4..63ed560b30 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java @@ -80,13 +80,13 @@ public final class IngestJobConfigurator { } /** - * Launches ingest jobs for one or more data sources using the ingest job + * Launches ingest job for one or more data sources using the ingest job * settings for the specified context. * * @param dataSources The data sources to ingest. */ @Deprecated public void startIngestJobs(List dataSources) { - IngestManager.getInstance().startIngestJobs(dataSources, this.settings, true); + IngestManager.getInstance().startIngestJob(dataSources, this.settings, true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 6587a20d19..020c05812b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -28,9 +28,9 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJobContext { - private final IngestJob ingestJob; + private final DataSourceIngestJob ingestJob; - IngestJobContext(IngestJob ingestJob) { + IngestJobContext(DataSourceIngestJob ingestJob) { this.ingestJob = ingestJob; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3e3f70c473..3b791dbfce 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -56,16 +56,15 @@ public class IngestManager { private static IngestManager instance = null; /** - * The ingest manager assigns a unique ID to each ingest job and maintains a - * mapping of job IDs to jobs. + * The ingest manager maintains a mapping of ingest job IDs to ingest jobs. */ - private final AtomicLong nextJobId = new AtomicLong(0L); private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); /** - * Each runnable/callable task the ingest manager farms out to a thread pool - * is given a unique thread/task ID. + * Each runnable/callable task the ingest manager submits to its thread + * pools is given a unique thread/task ID. */ + // TODO: It is no longer necessary to have multiple thread pools. private final AtomicLong nextThreadId = new AtomicLong(0L); /** @@ -214,14 +213,15 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for each data - * source in a collection of data sources. + * Starts an ingest job for a collection of data sources. * * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param doMessageBoxes Whether or not to display message boxes for errors. + * @param doUI Whether or not to support user interaction, e.g., showing + * message boxes and reporting progress through the NetBeans Progress API. + * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doMessageBoxes) { + public synchronized void startIngestJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -230,9 +230,17 @@ public class IngestManager { ingestMonitor.start(); } - long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, doMessageBoxes)); - ingestJobStarters.put(taskId, task); + if (doUI) { + /** + * Assume request is from code running on the EDT and dispatch to a + * worker thread. + */ + long taskId = nextThreadId.incrementAndGet(); + Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, dataSources, settings, doUI)); + ingestJobStarters.put(taskId, task); + } else { + this.startJob(dataSources, settings); + } } /** @@ -241,6 +249,8 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { + // RJCTODO: This may return the wrong answer if an IngestJobStarter has + // been dispatched to the startIngestJobsThreadPool. return !this.jobsById.isEmpty(); } @@ -405,19 +415,19 @@ public class IngestManager { } /** - * Starts an ingest job for a data source. + * Starts an ingest job for a collection of data sources. * - * @param dataSource The data source to ingest. + * @param dataSource The data sources to ingest. * @param settings The settings for the job. * @return A collection of ingest module start up errors, empty on success. */ - private List startJob(Content dataSource, IngestJobSettings settings) { + private List startJob(Collection dataSources, IngestJobSettings settings) { List errors = new ArrayList<>(); if (this.jobCreationIsEnabled) { - long jobId = this.nextJobId.incrementAndGet(); - IngestJob job = new IngestJob(jobId, dataSource, settings); + IngestJob job = new IngestJob(dataSources, settings); + long jobId = job.getId(); this.jobsById.put(jobId, job); - errors = job.start(settings.getEnabledIngestModuleTemplates()); + errors = job.start(); if (errors.isEmpty() && job.hasIngestPipeline()) { this.fireIngestJobStarted(jobId); IngestManager.logger.log(Level.INFO, "Ingest job {0} started", jobId); @@ -627,19 +637,18 @@ public class IngestManager { * * @return A list of ingest job state snapshots. */ - List getIngestJobSnapshots() { - List snapShots = new ArrayList<>(); + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); for (IngestJob job : this.jobsById.values()) { - snapShots.add(job.getSnapshot()); + snapShots.addAll(job.getDetailedSnapshot()); } return snapShots; } /** - * Creates and starts an ingest job, i.e., processing by ingest modules, for - * each data source in a collection of data sources. + * Creates and starts an ingest job for a collection of data sources. */ - private final class IngestJobsStarter implements Callable { + private final class IngestJobStarter implements Callable { private final long threadId; private final Collection dataSources; @@ -647,7 +656,7 @@ public class IngestManager { private final boolean doStartupErrorsMsgBox; private ProgressHandle progress; - IngestJobsStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { + IngestJobStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { this.threadId = threadId; this.dataSources = dataSources; this.settings = settings; @@ -683,61 +692,35 @@ public class IngestManager { return true; } }); - progress.start(dataSources.size()); + progress.start(); /** - * Try to start the ingest jobs. + * Try to start the ingest job. */ - int workUnitsCompleted = 0; - for (Content dataSource : this.dataSources) { - - /** - * Cancellation check. - */ - if (Thread.currentThread().isInterrupted()) { - return null; + List errors = IngestManager.this.startJob(this.dataSources, this.settings); + if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); } - - /** - * Add a "subtitle" to the display name of the progress bar - * to indicate an ingest job is being started for this data - * source. - */ - String progressMessage = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.progressingDisplayName", - dataSource.getName()); - progress.progress(progressMessage); - - /** - * Start an ingest job for this data source. - */ - List errors = IngestManager.this.startJob(dataSource, this.settings); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); - } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - } - - progress.progress(progressMessage, ++workUnitsCompleted); + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); } return null; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java index 0e59b3ceb5..90ee2b6078 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java @@ -37,8 +37,7 @@ public final class IngestProgressSnapshotDialog extends JDialog { private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestProgressSnapshotDialog.title.text"); private static final Dimension DIMENSIONS = new Dimension(500, 300); - private JDialog pseudoOwner = null; - + /** * Constructs a non-modal instance of the dialog with its own frame. */ @@ -54,8 +53,8 @@ public final class IngestProgressSnapshotDialog extends JDialog { */ public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) { super((Window) owner, TITLE, ModalityType.MODELESS); - if (shouldBeModal) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. - pseudoOwner = (JDialog) owner; + if (shouldBeModal && owner instanceof JDialog) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. + final JDialog pseudoOwner = (JDialog) owner; addWindowListener(new WindowAdapter() { @Override public void windowClosed(WindowEvent e) { // Put it back to how it was before we manipulated it. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 0d05337499..0e9e97a96b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -163,7 +163,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final String[] columnNames = {"Job ID", "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; - private List jobSnapshots; + private List jobSnapshots; private IngestJobTableModel() { refresh(); @@ -191,7 +191,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { - IngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); + DataSourceIngestJob.Snapshot snapShot = jobSnapshots.get(rowIndex); Object cellValue; switch (columnIndex) { case 0: diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 7ff6634f64..cfef33be7a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -23,15 +23,15 @@ import org.sleuthkit.datamodel.Content; abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; - private final IngestJob job; + private final DataSourceIngestJob job; private long threadId; - IngestTask(IngestJob job) { + IngestTask(DataSourceIngestJob job) { this.job = job; threadId = NOT_SET; } - IngestJob getIngestJob() { + DataSourceIngestJob getIngestJob() { return job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index d401b1439c..2adfb3ca40 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -148,7 +148,7 @@ final class IngestTasksScheduler { * @throws InterruptedException if the calling thread is blocked due to a * full tasks queue and is interrupted. */ - synchronized void scheduleIngestTasks(IngestJob job) { + synchronized void scheduleIngestTasks(DataSourceIngestJob job) { // Scheduling of both a data source ingest task and file ingest tasks // for a job must be an atomic operation. Otherwise, the data source // task might be completed before the file tasks are scheduled, @@ -163,7 +163,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleDataSourceIngestTask(IngestJob job) { + synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) { DataSourceIngestTask task = new DataSourceIngestTask(job); this.tasksInProgress.add(task); try { @@ -183,7 +183,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleFileIngestTasks(IngestJob job) { + synchronized void scheduleFileIngestTasks(DataSourceIngestJob job) { // Get the top level files for the data source associated with this job // and add them to the root directories priority queue. List topLevelFiles = getTopLevelFiles(job.getDataSource()); @@ -203,7 +203,7 @@ final class IngestTasksScheduler { * @param job The job for which the tasks are to be scheduled. * @param file The file to be associated with the task. */ - synchronized void scheduleFileIngestTask(IngestJob job, AbstractFile file) { + synchronized void scheduleFileIngestTask(DataSourceIngestJob job, AbstractFile file) { FileIngestTask task = new FileIngestTask(job, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { this.tasksInProgress.add(task); @@ -228,7 +228,7 @@ final class IngestTasksScheduler { * @param job The job for which the query is to be performed. * @return True or false. */ - synchronized boolean tasksForJobAreCompleted(IngestJob job) { + synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) { for (IngestTask task : tasksInProgress) { if (task.getIngestJob().getId() == job.getId()) { return false; @@ -245,7 +245,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to to canceled. */ - synchronized void cancelPendingTasksForIngestJob(IngestJob job) { + synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { long jobId = job.getId(); this.removeTasksForJob(this.rootDirectoryTasks, jobId); this.removeTasksForJob(this.directoryTasks, jobId); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index d8a7536ac1..b69e967ad9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -199,7 +199,7 @@ public final class RunIngestModulesDialog extends JDialog { ingestJobSettings.save(); showWarnings(ingestJobSettings); if (startIngestJob) { - IngestManager.getInstance().startIngestJobs(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); + IngestManager.getInstance().startIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); } setVisible(false); dispose(); diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index b808dd0d96..575a34a2f6 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 17:10:10 -0400 +#Mon, 15 Dec 2014 16:14:04 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index f0109bc92b..9ca23c2a58 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 12:13:36 -0400 +#Mon, 15 Dec 2014 16:14:04 -0500 CTL_MainWindow_Title=Autopsy 3.1.1 CTL_MainWindow_Title_No_Project=Autopsy 3.1.1