newCompletedJobsList = new ArrayList<>();
+ private Lock currentDirLock;
/**
* Searches the input directories for manifest files. The search results
@@ -1227,9 +1121,9 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
} catch (Exception ex) {
/*
- * NOTE: Need to catch all exceptions here. Otherwise
- * uncaught exceptions will propagate up to the calling
- * thread and may stop it from running.
+ * NOTE: Need to catch all unhandled exceptions here.
+ * Otherwise uncaught exceptions will propagate up to the
+ * calling thread and may stop it from running.
*/
sysLogger.log(Level.SEVERE, String.format("Error scanning the input directory %s", rootInputDirectory), ex);
}
@@ -1263,20 +1157,15 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
/**
- * Invoked for a file in a directory. If the file is a manifest file,
- * creates a pending pending or completed auto ingest job for the
- * manifest, based on the data stored in the coordination service node
- * for the manifest.
- *
- * Note that the mapping of case names to manifest paths that is used
- * for case deletion is updated as well.
+ * Creates a pending or completed auto ingest job if the file visited is
+ * a manifest file, based on the data stored in the coordination service
+ * node for the manifest.
*
* @param filePath The path of the file.
* @param attrs The file system attributes of the file.
*
- * @return TERMINATE if auto ingest is shutting down, CONTINUE if it has
- * not.
- *
+ * @return TERMINATE if auto ingest is shutting down, CONTINUE
+ * otherwise.
*/
@Override
public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) {
@@ -1285,6 +1174,10 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
try {
+ /*
+ * Determine whether or not the file is a manifest file. If it
+ * is, then parse it.
+ */
Manifest manifest = null;
for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) {
if (parser.fileIsManifest(filePath)) {
@@ -1304,125 +1197,111 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
return TERMINATE;
}
- if (null != manifest) {
- /*
- * Update the mapping of case names to manifest paths that
- * is used for case deletion.
- */
- String caseName = manifest.getCaseName();
- Path manifestPath = manifest.getFilePath();
- if (casesToManifests.containsKey(caseName)) {
- Set manifestPaths = casesToManifests.get(caseName);
- manifestPaths.add(manifestPath);
- } else {
- Set manifestPaths = new HashSet<>();
- manifestPaths.add(manifestPath);
- casesToManifests.put(caseName, manifestPaths);
- }
-
- /*
- * Add a job to the pending jobs queue, the completed jobs
- * list, or do crashed job recovery, as required.
- */
- try {
- byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString());
- if (null != rawData && rawData.length > 0) {
- try {
- AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData);
- AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus();
- switch (processingStatus) {
- case PENDING:
- addPendingJob(manifest, nodeData);
- break;
- case PROCESSING:
- doRecoveryIfCrashed(manifest, nodeData);
- break;
- case COMPLETED:
- addCompletedJob(manifest, nodeData);
- break;
- case DELETED:
- /*
- * Ignore jobs marked as "deleted."
- */
- break;
- default:
- sysLogger.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus");
- break;
- }
- } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) {
- sysLogger.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex);
- }
- } else {
- try {
- addNewPendingJob(manifest);
- } catch (AutoIngestJobException ex) {
- sysLogger.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex);
- }
- }
- } catch (CoordinationServiceException ex) {
- sysLogger.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex);
- return CONTINUE;
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- return TERMINATE;
+ if (manifest == null) {
+ return CONTINUE;
+ }
+ /*
+ * If a manifest file has been found, get the corresponding auto
+ * ingest job state from the manifest file coordination service
+ * node and put the job in the appropriate jobs list.
+ *
+ * There can be a race condition between queuing jobs and case
+ * deletion. However, in practice eliminating the race condition
+ * by acquiring a manifest file coordination service lock when
+ * analyzing job state here appears to have a significant
+ * performance cost for both input directory scanning and
+ * dequeuing jobs. Therefore, job state must be checked again
+ * during job dequeuing, while actually holding the lock, before
+ * executing the job.
+ */
+ String manifestFilePath = manifest.getFilePath().toString();
+ byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestFilePath);
+ if (null != rawData && rawData.length > 0) {
+ AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData);
+ AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus();
+ switch (processingStatus) {
+ case PENDING:
+ addPendingJob(manifest, nodeData);
+ break;
+ case PROCESSING:
+ doRecoveryIfCrashed(manifest, nodeData);
+ break;
+ case COMPLETED:
+ addCompletedJob(manifest, nodeData);
+ break;
+ case DELETED:
+ break;
+ default:
+ sysLogger.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus");
+ break;
}
+ } else {
+ addNewPendingJob(manifest);
}
+ } catch (CoordinationServiceException | AutoIngestJobException | AutoIngestJobNodeData.InvalidDataException ex) {
+ sysLogger.log(Level.SEVERE, String.format("Error visiting %s", filePath), ex);
+
+ } catch (InterruptedException ex) {
+ return TERMINATE;
+
} catch (Exception ex) {
- // Catch all unhandled and unexpected exceptions. Otherwise one bad file
- // can stop the entire input folder scanning. Given that the exception is unexpected,
- // I'm hesitant to add logging which requires accessing or de-referencing data.
- sysLogger.log(Level.SEVERE, "Unexpected exception in file visitor", ex);
- return CONTINUE;
+ /*
+ * This is an exception firewall so that an unexpected runtime
+ * exception from the handling of a single file does not stop
+ * the input directory scan.
+ */
+ sysLogger.log(Level.SEVERE, String.format("Unexpected exception visiting %s", filePath), ex);
}
if (!Thread.currentThread().isInterrupted()) {
return CONTINUE;
} else {
+ sysLogger.log(Level.WARNING, String.format("Auto ingest shut down while visiting %s", filePath));
return TERMINATE;
}
}
/**
- * Adds an existing job to the pending jobs queue.
+ * Adds an existing auto ingest job to the pending jobs queue. If the
+ * version of the coordination service node data is out of date, it is
+ * upgraded to the current version.
*
* @param manifest The manifest for the job.
- * @param nodeData The data stored in the coordination service node for
- * the job.
+ * @param nodeData The data stored in the manifest file coordination
+ * service node for the job.
*
- * @throws InterruptedException if the thread running the input
- * directory scan task is interrupted while
- * blocked, i.e., if auto ingest is
- * shutting down.
+ * @throws AutoIngestJobException If there was an error working with the
+ * node data.
+ * @throws InterruptedException If the thread running the input
+ * directory scan task is interrupted
+ * while blocked, i.e., if auto ingest is
+ * shutting down.
*/
- private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException {
+ private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws AutoIngestJobException, InterruptedException {
AutoIngestJob job;
if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) {
job = new AutoIngestJob(nodeData);
} else {
+ /*
+ * Upgrade the auto ingest node data to the current version.
+ */
job = new AutoIngestJob(manifest);
- job.setPriority(nodeData.getPriority()); // Retain priority, present in all versions of the node data.
+ job.setPriority(nodeData.getPriority());
Path caseDirectory = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName());
if (null != caseDirectory) {
job.setCaseDirectoryPath(caseDirectory);
}
/*
- * Try to upgrade/update the coordination service manifest node
- * data for the job.
- *
- * An exclusive lock is obtained before doing so because another
- * host may have already found the job, obtained an exclusive
- * lock, and started processing it. However, this locking does
- * make it possible that two processing hosts will both try to
- * obtain the lock to do the upgrade operation at the same time.
- * If this happens, the host that is holding the lock will
- * complete the upgrade operation, so there is nothing more for
- * this host to do.
+ * Try to write the upgraded node data to coordination service
+ * manifest node data for the job. If the lock cannot be
+ * obtained, assume that the auto ingest node holding the lock
+ * is taking care of this.
*/
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) {
- updateCoordinationServiceManifestNode(job);
+ updateAutoIngestJobData(job);
}
} catch (CoordinationServiceException ex) {
sysLogger.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex);
@@ -1432,154 +1311,145 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
/**
- * Adds a new job to the pending jobs queue.
+ * Adds a new auto ingest job to the pending jobs queue.
*
* @param manifest The manifest for the job.
*
- * @throws InterruptedException if the thread running the input
- * directory scan task is interrupted while
- * blocked, i.e., if auto ingest is
- * shutting down.
+ * @throws AutoIngestJobException If there was an error creating
+ * the node data.
+ * @throws CoordinationServiceException If there was an error writing
+ * the node data by the
+ * coordination service.
+ * @throws InterruptedException If the thread running the input
+ * directory scan task is
+ * interrupted while blocked, i.e.,
+ * if auto ingest is shutting down.
*/
- private void addNewPendingJob(Manifest manifest) throws InterruptedException, AutoIngestJobException {
+ private void addNewPendingJob(Manifest manifest) throws AutoIngestJobException, CoordinationServiceException, InterruptedException {
/*
- * Create the coordination service manifest node data for the job.
- * Note that getting the lock will create the node for the job (with
- * no data) if it does not already exist.
- *
- * An exclusive lock is obtained before creating the node data
- * because another host may have already found the job, obtained an
- * exclusive lock, and started processing it. However, this locking
- * does make it possible that two hosts will both try to obtain the
- * lock to do the create operation at the same time. If this
- * happens, the host that is locked out will not add the job to its
- * pending queue for this scan of the input directory, but it will
- * be picked up on the next scan.
+ * Create the coordination service manifest file node data for the
+ * job. Getting the lock both guards the writing of the new node
+ * data and creates the coordination service node if it does not
+ * already exist. Note that if this auto ingest node cannot get the
+ * lock, it is assumed that the auto ingest node holding the lock is
+ * taking care of this. In this case, this auto ingest node will not
+ * add the new job to its pending queue during this scan of the
+ * input directory, but it will be picked up during the next scan.
*/
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) {
AutoIngestJob job = new AutoIngestJob(manifest);
- updateCoordinationServiceManifestNode(job);
+ updateAutoIngestJobData(job);
newPendingJobsList.add(job);
}
- } catch (CoordinationServiceException ex) {
- sysLogger.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex);
}
}
/**
- * Does crash recovery for a manifest, if required. The criterion for
- * crash recovery is a manifest with coordination service node data
- * indicating it is being processed for which an exclusive lock on the
- * node can be acquired. If this condition is true, it is probable that
- * the node that was processing the job crashed and the processing
- * status was not updated.
+ * If required, does recovery for an auto ingest job that was left in
+ * the processing state by an auto ingest node (AIN) that crashed.
*
- * @param manifest The manifest for upgrading the node.
- * @param jobNodeData The auto ingest job node data.
+ * @param manifest The manifest for the job.
+ * @param nodeData The data stored in the manifest file lock
+ * coordination service node for the job.
*
- * @throws InterruptedException if the thread running the input
- * directory scan task is interrupted
- * while blocked, i.e., if auto ingest is
- * shutting down.
- * @throws AutoIngestJobException if there is an issue creating a new
- * AutoIngestJob object.
+ * @throws AutoIngestJobException If there was an error working
+ * with the node data.
+ * @throws CoordinationServiceException If there was an error writing
+ * updated node data by the
+ * coordination service.
+ * @throws InterruptedException If the thread running the input
+ * directory scan task is
+ * interrupted while blocked, i.e.,
+ * if auto ingest is shutting down.
*/
- private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData jobNodeData) throws InterruptedException, AutoIngestJobException {
- /*
- * Try to get an exclusive lock on the coordination service node for
- * the job. If the lock cannot be obtained, another host in the auto
- * ingest cluster is already doing the recovery, so there is nothing
- * to do.
- */
+ private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData jobNodeData) throws AutoIngestJobException, CoordinationServiceException, InterruptedException {
String manifestPath = manifest.getFilePath().toString();
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifestPath)) {
if (null != manifestLock) {
- sysLogger.log(Level.SEVERE, "Attempting crash recovery for {0}", manifestPath);
- Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName());
-
- /*
- * Create the recovery job.
- */
AutoIngestJob job = new AutoIngestJob(jobNodeData);
- int numberOfCrashes = job.getNumberOfCrashes();
- if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
+ if (job.getProcessingStatus() == AutoIngestJob.ProcessingStatus.PROCESSING) {
+ /*
+ * If the lock can be obtained with the job status set
+ * to processing, then an auto ingest node crashed while
+ * executing the job and was unable to update the job
+ * status.
+ */
+ sysLogger.log(Level.SEVERE, "Attempting crash recovery for {0}", manifestPath);
+
+ /*
+ * First, try to set the case node data error flag that
+ * indicates there was an auto ingest job error. If the
+ * auto ingest node that was executing the job crashed
+ * before the case directory was created, the job was a
+ * no-op, so the error flag does not need to be set.
+ * However, note that if another auto ingest job
+ * subsequently completed, the failed job may still have
+ * been a no-op, but in this case the flag will be set
+ * anyway, because a case directory will be found.
+ */
+ Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName());
+ if (null != caseDirectoryPath) {
+ job.setCaseDirectoryPath(caseDirectoryPath);
+ job.setErrorsOccurred(true);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
+ } else {
+ job.setErrorsOccurred(false);
+ }
+
+ /*
+ * Update the crash count for the job, determine whether
+ * or not to retry processing its data source, and deal
+ * with the job accordingly.
+ */
+ int numberOfCrashes = job.getNumberOfCrashes();
++numberOfCrashes;
job.setNumberOfCrashes(numberOfCrashes);
- if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
+ if (numberOfCrashes < AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
+ job.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING);
job.setCompletedDate(new Date(0));
+ if (null != caseDirectoryPath) {
+ try {
+ new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryWithRetry();
+ } catch (AutoIngestJobLoggerException ex) {
+ sysLogger.log(Level.SEVERE, String.format("Error writing case auto ingest log entry for crashed job for %s", manifestPath), ex);
+ }
+ }
+ updateAutoIngestJobData(job);
+ newPendingJobsList.add(job);
} else {
+ job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
job.setCompletedDate(Date.from(Instant.now()));
- }
- }
-
- if (null != caseDirectoryPath) {
- job.setCaseDirectoryPath(caseDirectoryPath);
- job.setErrorsOccurred(true);
- try {
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
- } catch (IOException ex) {
- sysLogger.log(Level.SEVERE, String.format("Error attempting to set error flag in case node data for %s", caseDirectoryPath), ex);
- }
- } else {
- job.setErrorsOccurred(false);
- }
-
- if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
- job.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING);
- if (null != caseDirectoryPath) {
- try {
- new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryWithRetry();
- } catch (AutoIngestJobLoggerException ex) {
- sysLogger.log(Level.SEVERE, String.format("Error creating case auto ingest log entry for crashed job for %s", manifestPath), ex);
+ if (null != caseDirectoryPath) {
+ try {
+ new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryNoRetry();
+ } catch (AutoIngestJobLoggerException ex) {
+ sysLogger.log(Level.SEVERE, String.format("Error writing case auto ingest log entry for crashed job for %s", manifestPath), ex);
+ }
}
+ updateAutoIngestJobData(job);
+ newCompletedJobsList.add(job);
}
- } else {
- job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
- if (null != caseDirectoryPath) {
- try {
- new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryNoRetry();
- } catch (AutoIngestJobLoggerException ex) {
- sysLogger.log(Level.SEVERE, String.format("Error creating case auto ingest log entry for crashed job for %s", manifestPath), ex);
- }
- }
- }
-
- /*
- * Update the coordination service node for the job. If this
- * fails, leave the recovery to another host.
- */
- try {
- updateCoordinationServiceManifestNode(job);
- } catch (CoordinationServiceException ex) {
- sysLogger.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex);
- return;
- }
-
- jobNodeData = new AutoIngestJobNodeData(job);
-
- if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
- newPendingJobsList.add(job);
- } else {
- newCompletedJobsList.add(new AutoIngestJob(jobNodeData));
}
}
- } catch (CoordinationServiceException ex) {
- sysLogger.log(Level.SEVERE, String.format("Error attempting to get exclusive lock for %s", manifestPath), ex);
}
}
/**
* Adds a job to process a manifest to the completed jobs list.
*
- * @param nodeData The data stored in the coordination service node for
- * the manifest.
- * @param manifest The manifest for upgrading the node.
+ * @param manifest The manifest for the job.
+ * @param nodeData The data stored in the manifest file lock
+ * coordination service node for the job.
*
- * @throws CoordinationServiceException
- * @throws InterruptedException
+ * @throws AutoIngestJobException If there was an error working with the
+ * node data.
+ * @throws InterruptedException If the thread running the input
+ * directory scan task is interrupted
+ * while blocked, i.e., if auto ingest is
+ * shutting down.
*/
- private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException, AutoIngestJobException {
+ private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws AutoIngestJobException, InterruptedException {
Path caseDirectoryPath = nodeData.getCaseDirectoryPath();
if (!caseDirectoryPath.toFile().exists()) {
sysLogger.log(Level.WARNING, String.format("Job completed for %s, but cannot find case directory %s, ignoring job", nodeData.getManifestFilePath(), caseDirectoryPath.toString()));
@@ -1591,19 +1461,11 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
job = new AutoIngestJob(nodeData);
job.setCaseDirectoryPath(caseDirectoryPath);
} else {
- /**
- * Use the manifest rather than the node data here to create a
- * new AutoIngestJob instance because the AutoIngestJob
- * constructor that takes a node data object expects the node
- * data to have fields that do not exist in earlier versions.
+ /*
+ * Upgrade the auto ingest node data to the current version.
*/
job = new AutoIngestJob(manifest);
job.setCaseDirectoryPath(caseDirectoryPath);
-
- /**
- * Update the job with the fields that exist in all versions of
- * the nodeData.
- */
job.setCompletedDate(nodeData.getCompletedDate());
job.setErrorsOccurred(nodeData.getErrorsOccurred());
job.setPriority(nodeData.getPriority());
@@ -1612,20 +1474,20 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
/*
- * Try to upgrade/update the coordination service manifest node
- * data for the job. It is possible that two hosts will both try
- * to obtain the lock to do the upgrade operation at the same
- * time. If this happens, the host that is holding the lock will
- * complete the upgrade operation.
+ * Try to write the upgraded node data to coordination service
+ * manifest node data for the job. If the lock cannot be
+ * obtained, assume that the auto ingest node holding the lock
+ * is taking care of this.
*/
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) {
- updateCoordinationServiceManifestNode(job);
+ updateAutoIngestJobData(job);
}
} catch (CoordinationServiceException ex) {
sysLogger.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex);
}
}
+
newCompletedJobsList.add(job);
}
@@ -1654,17 +1516,17 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
/**
- * Invoked for an input directory after entries in the directory are
+ * Invoked for an input directory after the files in the directory are
* visited. Checks if the task thread has been interrupted because auto
* ingest is shutting down and terminates the scan if that is the case.
*
* @param dirPath The directory about to be visited.
* @param unused Unused.
*
- * @return TERMINATE if the task thread has been interrupted, CONTINUE
- * if it has not.
+ * @return FileVisitResult.TERMINATE if the task thread has been
+ * interrupted, FileVisitResult.CONTINUE if it has not.
*
- * @throws IOException if an I/O error occurs, but this implementation
+ * @throws IOException If an I/O error occurs, but this implementation
* does not throw.
*/
@Override
@@ -2121,17 +1983,40 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
try {
+ /*
+ * There can be a race condition between queuing jobs
+ * and case deletion. However, in practice eliminating
+ * the race condition by acquiring a manifest file
+ * coordination service lock when analyzing job state
+ * during the input directory scan appears to have a
+ * significant performance cost for both input directory
+ * scanning and dequeuing jobs. Therefore, job state
+ * must be checked again here, while actually holding
+ * the lock, before executing the job.
+ */
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()));
if (!nodeData.getProcessingStatus().equals(PENDING)) {
- /*
- * Due to a timing issue or a missed event, a
- * non-pending job has ended up on the pending
- * queue. Skip the job and remove it from the queue.
- */
iterator.remove();
+ manifestLock.release();
+ manifestLock = null;
continue;
}
+ /*
+ * Ditto for the presence of the manifest file.
+ */
+ File manifestFile = nodeData.getManifestFilePath().toFile();
+ if (!manifestFile.exists()) {
+ iterator.remove();
+ manifestLock.release();
+ manifestLock = null;
+ continue;
+ }
+
+ /*
+ * Finally, check for devoting too many resources to a
+ * single case, if the check is enabled.
+ */
if (enforceMaxJobsPerCase) {
int currentJobsForCase = 0;
for (AutoIngestJob runningJob : hostNamesToRunningJobs.values()) {
@@ -2145,11 +2030,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
continue;
}
}
+
iterator.remove();
currentJob = job;
break;
+
} catch (AutoIngestJobNodeData.InvalidDataException ex) {
- sysLogger.log(Level.WARNING, String.format("Unable to use node data for %s", manifestPath), ex); // JCTODO: Is this right?
+ sysLogger.log(Level.WARNING, String.format("Unable to use node data for %s", manifestPath), ex);
}
}
}
@@ -2220,7 +2107,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PROCESSING);
currentJob.setProcessingStage(AutoIngestJob.Stage.STARTING, Date.from(Instant.now()));
currentJob.setProcessingHostName(AutoIngestManager.LOCAL_HOST_NAME);
- updateCoordinationServiceManifestNode(currentJob);
+ updateAutoIngestJobData(currentJob);
setChanged();
notifyObservers(Event.JOB_STARTED);
eventPublisher.publishRemotely(new AutoIngestJobStartedEvent(currentJob));
@@ -2244,14 +2131,14 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING);
}
currentJob.setProcessingHostName("");
- updateCoordinationServiceManifestNode(currentJob);
+ updateAutoIngestJobData(currentJob);
boolean retry = (!currentJob.isCanceled() && !currentJob.isCompleted());
sysLogger.log(Level.INFO, "Completed processing of {0}, retry = {1}", new Object[]{manifestPath, retry});
if (currentJob.isCanceled()) {
Path caseDirectoryPath = currentJob.getCaseDirectoryPath();
if (null != caseDirectoryPath) {
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, currentJob.getManifest().getDataSourceFileName(), caseDirectoryPath);
jobLogger.logJobCancelled();
}
@@ -2410,9 +2297,9 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
sysLogger.log(Level.INFO, "Opening case {0} for {1}", new Object[]{caseName, manifest.getFilePath()});
currentJob.setProcessingStage(AutoIngestJob.Stage.OPENING_CASE, Date.from(Instant.now()));
/*
- * Acquire and hold a case name lock so that only one node at as
- * time can scan the output directory at a time. This prevents
- * making duplicate cases for the saem auto ingest case.
+ * Acquire and hold a case name lock so that only one node at a time
+ * can search the output directory for an existing case. This
+ * prevents making duplicate cases for the same auto ingest case.
*/
try (Lock caseLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.CASES, caseName, 30, TimeUnit.MINUTES)) {
if (null != caseLock) {
@@ -2440,13 +2327,16 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
Thread.sleep(AutoIngestUserPreferences.getSecondsToSleepBetweenCases() * 1000);
}
currentJob.setCaseDirectoryPath(caseDirectoryPath);
- updateCoordinationServiceManifestNode(currentJob); // update case directory path
+ updateAutoIngestJobData(currentJob);
+ recordManifest(caseDirectoryPath, manifest.getFilePath());
Case caseForJob = Case.getCurrentCase();
sysLogger.log(Level.INFO, "Opened case {0} for {1}", new Object[]{caseForJob.getName(), manifest.getFilePath()});
return caseForJob;
} catch (KeywordSearchModuleException ex) {
throw new CaseManagementException(String.format("Error creating solr settings file for case %s for %s", caseName, manifest.getFilePath()), ex);
+ } catch (IOException ex) {
+ throw new CaseManagementException(String.format("Error recording manifest file path for case %s for %s", caseName, manifest.getFilePath()), ex);
} catch (CaseActionException ex) {
throw new CaseManagementException(String.format("Error creating or opening case %s for %s", caseName, manifest.getFilePath()), ex);
}
@@ -2456,6 +2346,22 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
}
+ /**
+ * Writes the path of the manifest file for the current job to a list of
+ * manifest file paths for the case in file in the case directory.
+ *
+ * @param caseDirectoryPath The case directory path.
+ *
+ * @throws IOException If the file cannot be created or opened and
+ * updated.
+ */
+ private void recordManifest(Path caseDirectoryPath, Path manifestFilePath) throws IOException {
+ final Path manifestsListFilePath = Paths.get(caseDirectoryPath.toString(), AutoIngestManager.getCaseManifestsListFileName());
+ try (FileWriter fileWriter = new FileWriter(manifestsListFilePath.toString(), true)) {
+ fileWriter.write(manifestFilePath.toString() + "\n");
+ }
+ }
+
/**
* Runs the ingest process for the current job.
*
@@ -2590,7 +2496,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
if (!dataSource.exists()) {
sysLogger.log(Level.SEVERE, "Missing data source for {0}", manifestPath);
currentJob.setErrorsOccurred(true);
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logMissingDataSource();
return null;
}
@@ -2635,7 +2541,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
// did we find a data source processor that can process the data source
if (validDataSourceProcessors.isEmpty()) {
// This should never happen. We should add all unsupported data sources as logical files.
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
currentJob.setErrorsOccurred(true);
jobLogger.logFailedToIdentifyDataSource();
sysLogger.log(Level.WARNING, "Unsupported data source {0} for {1}", new Object[]{dataSource.getPath(), manifestPath}); // NON-NLS
@@ -2670,7 +2576,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
// If we get to this point, none of the processors were successful
sysLogger.log(Level.SEVERE, "All data source processors failed to process {0}", dataSource.getPath());
jobLogger.logFailedToAddDataSource();
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
currentJob.setErrorsOccurred(true);
// Throw an exception. It will get caught & handled upstream and will result in AIM auto-pause.
throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Failed to process " + dataSource.getPath() + " with all data source processors");
@@ -2789,7 +2695,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
if (!cancelledModules.isEmpty()) {
sysLogger.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath));
currentJob.setErrorsOccurred(true);
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
for (String module : snapshot.getCancelledDataSourceIngestModules()) {
sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath));
nestedJobLogger.logIngestModuleCancelled(module);
@@ -2799,7 +2705,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
} else {
currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now()));
currentJob.setErrorsOccurred(true);
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
nestedJobLogger.logAnalysisCancelled();
CancellationReason cancellationReason = snapshot.getCancellationReason();
if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) {
@@ -2812,13 +2718,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable());
}
currentJob.setErrorsOccurred(true);
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logIngestModuleStartupErrors();
throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath));
} else {
sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException());
currentJob.setErrorsOccurred(true);
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logAnalysisStartupError();
throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException());
}
@@ -2827,7 +2733,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning});
}
currentJob.setErrorsOccurred(true);
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logIngestJobSettingsErrors();
throw new AnalysisStartupException("Error(s) in ingest job settings");
}
@@ -2910,7 +2816,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
} catch (FileExportException ex) {
sysLogger.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex);
currentJob.setErrorsOccurred(true);
- setCaseNodeDataErrorsOccurred(caseDirectoryPath);
+ setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logFileExportError();
}
}
@@ -3096,7 +3002,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
currentJob.setModuleRuntimesSnapshot(IngestManager.getInstance().getModuleRunTimes());
setChanged();
notifyObservers(Event.JOB_STATUS_UPDATED);
- updateCoordinationServiceManifestNode(currentJob);
+ updateAutoIngestJobData(currentJob);
eventPublisher.publishRemotely(new AutoIngestJobStatusEvent(currentJob));
}
}
@@ -3256,12 +3162,6 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
- enum CaseDeletionResult {
- FAILED,
- PARTIALLY_DELETED,
- FULLY_DELETED
- }
-
static final class AutoIngestManagerException extends Exception {
private static final long serialVersionUID = 1L;
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMetricsCollector.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMetricsCollector.java
index 7b07a15aec..3192e55b7a 100644
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMetricsCollector.java
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMetricsCollector.java
@@ -72,13 +72,13 @@ final class AutoIngestMetricsCollector {
switch (processingStatus) {
case PENDING:
case PROCESSING:
- case DELETED:
/*
* These are not jobs we care about for metrics, so
* we will ignore them.
*/
break;
case COMPLETED:
+ case DELETED: // Assuming deleted jobs were completed before they were deleted.
newMetricsSnapshot.addCompletedJobMetric(job.getCompletedDate(), job.getDataSourceSize());
break;
default:
@@ -96,7 +96,7 @@ final class AutoIngestMetricsCollector {
return newMetricsSnapshot;
- } catch (CoordinationService.CoordinationServiceException ex) {
+ } catch (CoordinationService.CoordinationServiceException | InterruptedException ex) {
LOGGER.log(Level.SEVERE, "Failed to get node list from coordination service", ex);
return new MetricsSnapshot();
}
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java
index 8f7a9c0696..b3d3fb21a8 100644
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java
@@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.experimental.autoingest;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
-import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
@@ -38,20 +37,15 @@ import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.stream.Collectors;
import javax.annotation.concurrent.GuardedBy;
-import org.sleuthkit.autopsy.casemodule.Case;
-import org.sleuthkit.autopsy.casemodule.CaseActionException;
-import org.sleuthkit.autopsy.casemodule.CaseMetadata;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
-import org.sleuthkit.autopsy.coreutils.StopWatch;
import org.sleuthkit.autopsy.events.AutopsyEventException;
import org.sleuthkit.autopsy.events.AutopsyEventPublisher;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus;
import static org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus.DELETED;
import static org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus.PENDING;
-import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestManager.CaseDeletionResult;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestManager.Event;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeControlEvent.ControlEventType;
@@ -361,6 +355,9 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
newJobsSnapshot.addOrReplaceCompletedJob(job);
break;
case DELETED:
+ /*
+ * Ignore jobs marked as deleted.
+ */
break;
default:
LOGGER.log(Level.SEVERE, "Unknown AutoIngestJobData.ProcessingStatus");
@@ -378,7 +375,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
return newJobsSnapshot;
- } catch (CoordinationServiceException ex) {
+ } catch (CoordinationServiceException | InterruptedException ex) {
LOGGER.log(Level.SEVERE, "Failed to get node list from coordination service", ex);
return new JobsSnapshot();
}
@@ -659,88 +656,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
}
}
- /**
- * Deletes a case. This includes deleting the case directory, the text
- * index, and the case database. This does not include the directories
- * containing the data sources and their manifests.
- *
- * @param job The job whose case you want to delete
- *
- * @return A result code indicating success, partial success, or failure.
- */
- CaseDeletionResult deleteCase(AutoIngestJob job) {
- String caseName = job.getManifest().getCaseName();
- Path caseDirectoryPath = job.getCaseDirectoryPath();
- Path metadataFilePath = caseDirectoryPath.resolve(caseName + CaseMetadata.getFileExtension());
- StopWatch stopWatch = new StopWatch();
- stopWatch.start();
- synchronized (jobsLock) {
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to acquire jobsLock (Java monitor in AutoIngestMonitor class) for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath));
- stopWatch.reset();
- stopWatch.start();
- try {
- CaseMetadata metadata = new CaseMetadata(metadataFilePath);
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to read case metadata for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath));
- stopWatch.reset();
- stopWatch.start();
- Case.deleteCase(metadata);
- } catch (CaseMetadata.CaseMetadataException ex) {
- LOGGER.log(Level.SEVERE, String.format("Failed to read case metadata file %s for case %s at %s", metadataFilePath, caseName, caseDirectoryPath), ex);
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to fail to read case metadata file %s for case %s at %s", stopWatch.getElapsedTimeSecs(), metadataFilePath, caseName, caseDirectoryPath));
- return CaseDeletionResult.FAILED;
- } catch (CaseActionException ex) {
- LOGGER.log(Level.SEVERE, String.format("Failed to delete case %s at %s", caseName, caseDirectoryPath), ex);
- return CaseDeletionResult.FAILED;
- }
-
- // Update the state of completed jobs associated with this case to indicate
- // that the case has been deleted
- stopWatch.reset();
- stopWatch.start();
- List completedJobs = getCompletedJobs();
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to get completed jobs listing for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath));
- stopWatch.reset();
- stopWatch.start();
- for (AutoIngestJob completedJob : completedJobs) {
- if (caseName.equals(completedJob.getManifest().getCaseName())) {
- try {
- completedJob.setProcessingStatus(DELETED);
- AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(completedJob);
- coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, completedJob.getManifest().getFilePath().toString(), nodeData.toArray());
- } catch (CoordinationServiceException | InterruptedException ex) {
- LOGGER.log(Level.SEVERE, String.format("Failed to update completed job node data for %s when deleting case %s at %s", completedJob.getManifest().getFilePath(), caseName, caseDirectoryPath), ex);
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to fail to update job node data for completed jobs for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath));
- return CaseDeletionResult.PARTIALLY_DELETED;
- }
- }
- }
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to update job node data for completed jobs for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath));
-
- // Remove jobs associated with this case from the completed jobs collection.
- stopWatch.reset();
- stopWatch.start();
- completedJobs.removeIf((AutoIngestJob completedJob)
- -> completedJob.getManifest().getCaseName().equals(caseName));
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to remove completed jobs for case %s at %s from current jobs snapshot", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath));
-
- // Publish a message to update auto ingest nodes.
- stopWatch.reset();
- stopWatch.start();
- eventPublisher.publishRemotely(new AutoIngestCaseDeletedEvent(caseName, LOCAL_HOST_NAME, AutoIngestManager.getSystemUserNameProperty()));
- stopWatch.stop();
- LOGGER.log(Level.INFO, String.format("Used %d s to publish job deletion event for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath));
- }
-
- return CaseDeletionResult.FULLY_DELETED;
- }
-
/**
* Send the given control event to the given node.
*
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/BackgroundTaskAction.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/BackgroundTaskAction.java
new file mode 100755
index 0000000000..1140b49cf1
--- /dev/null
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/BackgroundTaskAction.java
@@ -0,0 +1,80 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2019-2019 Basis Technology Corp.
+ * Contact: carrier sleuthkit org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.experimental.autoingest;
+
+import java.awt.event.ActionEvent;
+import java.util.concurrent.FutureTask;
+import javax.swing.AbstractAction;
+import org.sleuthkit.autopsy.progress.AppFrameProgressBar;
+import org.sleuthkit.autopsy.progress.ProgressIndicator;
+import org.sleuthkit.autopsy.progress.TaskCancellable;
+
+/**
+ * A base class for action classes that kick off a cancellable task that runs in
+ * a background thread and reports progress using an application frame progress
+ * bar.
+ */
+abstract class BackgroundTaskAction extends AbstractAction {
+
+ private static final long serialVersionUID = 1L;
+ private final String progressDisplayName;
+
+ /**
+ * Constructs the base class part of action classes that kick off a
+ * cancellable task that runs in a background thread and reports progress
+ * using an application frame progress bar.
+ *
+ * @param actionName The name of the action.
+ * @param progressDisplayName The display name for the progress bar.
+ */
+ BackgroundTaskAction(String actionName, String progressDisplayName) {
+ super(actionName);
+ this.progressDisplayName = progressDisplayName;
+ }
+
+ @Override
+ public void actionPerformed(ActionEvent event) {
+ final AppFrameProgressBar progress = new AppFrameProgressBar(progressDisplayName);
+ final TaskCancellable taskCanceller = new TaskCancellable(progress);
+ progress.setCancellationBehavior(taskCanceller);
+ final Runnable task = getTask(progress);
+ final FutureTask future = new FutureTask<>(task, null);
+ taskCanceller.setFuture(future);
+ new Thread(future).start();
+ }
+
+ /**
+ * Gets the background task to be executed. The task is expected to report
+ * its progress using the supplied progress indicator and to check for
+ * cancellation by checking to see if the thread it is running in has been
+ * interrupted.
+ *
+ * @param progress A progress indicator for the task.
+ *
+ * @return The Runnnable task.
+ */
+ abstract Runnable getTask(ProgressIndicator progress);
+
+ @Override
+ public BackgroundTaskAction clone() throws CloneNotSupportedException {
+ super.clone();
+ throw new CloneNotSupportedException();
+ }
+
+}
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties
index 0f074ca11c..f3081bef89 100644
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties
@@ -228,7 +228,6 @@ AutoIngestControlPanel.bnShowProgress.text=Ingest Progress
AutoIngestControlPanel.bnCancelJob.text=&Cancel Job
AutoIngestControlPanel.bnCancelModule.text=Cancel &Module
AutoIngestControlPanel.bnReprocessJob.text=Reprocess Job
-AutoIngestControlPanel.bnDeleteCase.text=&Delete Case
AutoIngestControlPanel.bnShowCaseLog.text=Show Case &Log
AutoIngestControlPanel.bnPause.text=Pause
AutoIngestControlPanel.bnRefresh.text=&Refresh
@@ -255,3 +254,6 @@ AinStatusDashboard.clusterMetricsButton.text=Auto Ingest &Metrics
AinStatusDashboard.nodeStatusTableTitle.text=Auto Ingest Nodes
AinStatusDashboard.healthMonitorButton.text=Health Monitor
CasesDashboardTopComponent.refreshButton.text=Refresh
+AutoIngestCasesDeletionDialog.jLabel1.text=Progress
+CasesDashboardTopComponent.deleteOrphanCaseNodesButton.text=Delete Orphan Case Znodes
+CasesDashboardTopComponent.deleteOrphanManifestNodesButton.text=Delete Orphan Manifest Znodes
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED
index a7336e26a6..38d8ee47a1 100755
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED
@@ -10,8 +10,6 @@ AinStatusNode.status.title=Status
AinStatusNode.status.unknown=Unknown
AutoIngestAdminActions.cancelJobAction.title=Cancel Job
AutoIngestAdminActions.cancelModuleAction.title=Cancel Module
-AutoIngestAdminActions.deleteCaseAction.error=Failed to delete case.
-AutoIngestAdminActions.deleteCaseAction.title=Delete Case
AutoIngestAdminActions.pause.title=Pause Node
AutoIngestAdminActions.progressDialogAction.title=Ingest Progress
AutoIngestAdminActions.reprocessJobAction.error=Failed to reprocess job
@@ -56,7 +54,6 @@ AutoIngestControlPanel.Cancelling=Cancelling...
AutoIngestControlPanel.completedTable.toolTipText=The Completed table shows all Jobs that have been processed already
AutoIngestControlPanel.ConfigLocked=The shared configuration directory is locked because upload from another node is in progress. \nIf this is an error, you can unlock the directory and then retry the upload.
AutoIngestControlPanel.ConfigLockedTitle=Configuration directory locked
-AutoIngestControlPanel.DeletionFailed=Deletion failed for job
AutoIngestControlPanel.EnableConfigurationSettings=Enable shared configuration from the options panel before uploading
AutoIngestControlPanel.errorMessage.caseDeprioritization=An error occurred when deprioritizing the case. Some or all jobs may not have been deprioritized.
AutoIngestControlPanel.errorMessage.casePrioritization=An error occurred when prioritizing the case. Some or all jobs may not have been prioritized.
@@ -167,9 +164,58 @@ CTL_AutoIngestDashboardOpenAction=Auto Ingest Dashboard
CTL_AutoIngestDashboardTopComponent=Auto Ingest Jobs
CTL_CasesDashboardAction=Multi-User Cases Dashboard
CTL_CasesDashboardTopComponent=Cases
-DeleteCaseInputDirectoriesAction.menuItemText=Delete Input Directories
-DeleteCasesAction.menuItemText=Delete Case and Jobs
-DeleteCasesForReprocessingAction.menuItemText=Delete for Reprocessing
+DeleteCaseAction.confirmationText=Are you sure you want to delete the following for the case(s):\n\tManifest file znodes\n\tCase database\n\tCore.properties file\n\tCase directory\n\tCase znodes
+DeleteCaseAction.menuItemText=Delete Case(s)
+DeleteCaseAction.progressDisplayName=Delete Case(s)
+DeleteCaseAction.taskName=app-input-and-output
+DeleteCaseInputAction.confirmationText=Are you sure you want to delete the following for the case(s):\n\tManifest files\n\tData sources\n
+DeleteCaseInputAction.menuItemText=Delete Input
+DeleteCaseInputAction.progressDisplayName=Delete Input
+DeleteCaseInputAction.taskName=input
+DeleteCaseInputAndOutputAction.confirmationText=Are you sure you want to delete the following for the case(s):\n\tManifest files\n\tData sources\n\tManifest file znodes\n\tCase database\n\tCore.properties file\n\tCase directory\n\tCase znodes
+DeleteCaseInputAndOutputAction.menuItemText=Delete Input and Output
+DeleteCaseInputAndOutputAction.progressDisplayName=Delete Input and Output
+DeleteCaseInputAndOutputAction.taskName=input-and-output
+DeleteCaseOutputAction.confirmationText=Are you sure you want to delete the following for the case(s):\n\tManifest file znodes\n\tCase database\n\tCore.properties file\n\tCase directory\n\tCase znodes
+DeleteCaseOutputAction.menuItemText=Delete Output
+DeleteCaseOutputAction.progressDisplayName=Delete Output
+DeleteCaseOutputAction.taskName=output
+DeleteCaseTask.progress.acquiringCaseDirLock=Acquiring exclusive case directory lock...
+DeleteCaseTask.progress.acquiringCaseNameLock=Acquiring exclusive case name lock...
+DeleteCaseTask.progress.acquiringManifestLocks=Acquiring exclusive manifest file locks...
+DeleteCaseTask.progress.connectingToCoordSvc=Connecting to the coordination service...
+DeleteCaseTask.progress.deletingCaseDirCoordSvcNode=Deleting case directory znode...
+DeleteCaseTask.progress.deletingCaseNameCoordSvcNode=Deleting case name znode...
+# {0} - data source path
+DeleteCaseTask.progress.deletingDataSource=Deleting data source {0}...
+DeleteCaseTask.progress.deletingJobLogLockNode=Deleting case auto ingest log znode...
+# {0} - manifest file path
+DeleteCaseTask.progress.deletingManifest=Deleting manifest file {0}...
+# {0} - manifest file path
+DeleteCaseTask.progress.deletingManifestFileNode=Deleting the manifest file znode for {0}...
+DeleteCaseTask.progress.deletingResourcesLockNode=Deleting case resources znode...
+DeleteCaseTask.progress.gettingManifestPaths=Getting manifest file paths...
+# {0} - manifest file path
+DeleteCaseTask.progress.lockingManifest=Locking manifest file {0}...
+DeleteCaseTask.progress.openingCaseDatabase=Opening the case database...
+DeleteCaseTask.progress.openingCaseMetadataFile=Opening case metadata file...
+# {0} - manifest file path
+DeleteCaseTask.progress.parsingManifest=Parsing manifest file {0}...
+# {0} - manifest file path
+DeleteCaseTask.progress.releasingManifestLock=Releasing lock on the manifest file {0}...
+DeleteCaseTask.progress.startMessage=Starting deletion...
+DeleteOrphanCaseNodesAction.progressDisplayName=Cleanup Case Znodes
+DeleteOrphanCaseNodesTask.progress.connectingToCoordSvc=Connecting to the coordination service
+# {0} - node path
+DeleteOrphanCaseNodesTask.progress.deletingOrphanedCaseNode=Deleting orphaned case znode {0}
+DeleteOrphanCaseNodesTask.progress.gettingCaseNodesListing=Querying coordination service for case znodes
+DeleteOrphanCaseNodesTask.progress.startMessage=Starting orphaned case znode cleanup
+DeleteOrphanManifestNodesAction.progressDisplayName=Cleanup Manifest File Znodes
+DeleteOrphanManifestNodesTask.progress.connectingToCoordSvc=Connecting to the coordination service
+# {0} - node path
+DeleteOrphanManifestNodesTask.progress.deletingOrphanedManifestNode=Deleting orphaned manifest file znode {0}
+DeleteOrphanManifestNodesTask.progress.gettingManifestNodes=Querying the coordination service for manifest file znodes
+DeleteOrphanManifestNodesTask.progress.startMessage=Starting orphaned manifest file znode cleanup
HINT_CasesDashboardTopComponent=This is an adminstrative dashboard for multi-user cases
OpenAutoIngestLogAction.deletedLogErrorMsg=The case auto ingest log has been deleted.
OpenAutoIngestLogAction.logOpenFailedErrorMsg=Failed to open case auto ingest log. See application log for details.
@@ -286,7 +332,6 @@ PrioritizationAction.prioritizeJobAction.error=Failed to prioritize job "%s".
PrioritizationAction.prioritizeJobAction.title=Prioritize Job
PrioritizedIconCellRenderer.notPrioritized.tooltiptext=This job has not been prioritized.
PrioritizedIconCellRenderer.prioritized.tooltiptext=This job has been prioritized. The most recently prioritized job should be processed next.
-ShowCaseDeletionStatusAction.menuItemText=Show Deletion Status
SingleUserCaseImporter.NonUniqueOutputFolder=Output folder not unique. Skipping
SingleUserCaseImporter.WillImport=Will import:
SingleUserCaseImporter.None=None
@@ -380,7 +425,6 @@ AutoIngestControlPanel.bnShowProgress.text=Ingest Progress
AutoIngestControlPanel.bnCancelJob.text=&Cancel Job
AutoIngestControlPanel.bnCancelModule.text=Cancel &Module
AutoIngestControlPanel.bnReprocessJob.text=Reprocess Job
-AutoIngestControlPanel.bnDeleteCase.text=&Delete Case
AutoIngestControlPanel.bnShowCaseLog.text=Show Case &Log
AutoIngestControlPanel.bnPause.text=Pause
AutoIngestControlPanel.bnRefresh.text=&Refresh
@@ -407,3 +451,6 @@ AinStatusDashboard.clusterMetricsButton.text=Auto Ingest &Metrics
AinStatusDashboard.nodeStatusTableTitle.text=Auto Ingest Nodes
AinStatusDashboard.healthMonitorButton.text=Health Monitor
CasesDashboardTopComponent.refreshButton.text=Refresh
+AutoIngestCasesDeletionDialog.jLabel1.text=Progress
+CasesDashboardTopComponent.deleteOrphanCaseNodesButton.text=Delete Orphan Case Znodes
+CasesDashboardTopComponent.deleteOrphanManifestNodesButton.text=Delete Orphan Manifest Znodes
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardCustomizer.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardCustomizer.java
index 498a4e67ad..3126becbfe 100755
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardCustomizer.java
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardCustomizer.java
@@ -31,23 +31,28 @@ import org.sleuthkit.autopsy.casemodule.multiusercasesbrowser.MultiUserCaseBrows
*/
final class CasesDashboardCustomizer implements MultiUserCaseBrowserCustomizer {
- private final DeleteCaseInputDirectoriesAction deleteCaseInputAction;
- private final DeleteCasesForReprocessingAction deleteCaseOutputAction;
- private final DeleteCasesAction deleteCaseAction;
+ private final DeleteCaseAction deleteCaseAction;
+ private final DeleteCaseInputAction deleteCaseInputAction;
+ private final DeleteCaseOutputAction deleteCaseOutputAction;
+ private final DeleteCaseInputAndOutputAction deleteCaseInputAndOutputAction;
/**
* Constructs a customizer for the multi-user case browser panel used in the
* administrative dashboard for auto ingest cases to present a tabular view
* of the multi-user cases known to the coordination service.
+ *
+ * @param executor An executor for tasks for actions that do work in the
+ * background.
*/
CasesDashboardCustomizer() {
/*
* These actions are shared by all nodes in order to support multiple
* selection.
*/
- deleteCaseInputAction = new DeleteCaseInputDirectoriesAction();
- deleteCaseOutputAction = new DeleteCasesForReprocessingAction();
- deleteCaseAction = new DeleteCasesAction();
+ deleteCaseAction = new DeleteCaseAction();
+ deleteCaseInputAction = new DeleteCaseInputAction();
+ deleteCaseOutputAction = new DeleteCaseOutputAction();
+ deleteCaseInputAndOutputAction = new DeleteCaseInputAndOutputAction();
}
@Override
@@ -56,6 +61,13 @@ final class CasesDashboardCustomizer implements MultiUserCaseBrowserCustomizer {
properties.add(Column.CREATE_DATE);
properties.add(Column.LAST_ACCESS_DATE);
properties.add(Column.DIRECTORY);
+ properties.add(Column.MANIFEST_FILE_ZNODES_DELETE_STATUS);
+ if (AutoIngestDashboard.extendedFeaturesAreEnabled()) {
+ properties.add(Column.DATA_SOURCES_DELETE_STATUS);
+ }
+ properties.add(Column.TEXT_INDEX_DELETE_STATUS);
+ properties.add(Column.CASE_DB_DELETE_STATUS);
+ properties.add(Column.CASE_DIR_DELETE_STATUS);
return properties;
}
@@ -76,10 +88,13 @@ final class CasesDashboardCustomizer implements MultiUserCaseBrowserCustomizer {
List actions = new ArrayList<>();
actions.add(new OpenCaseAction(nodeData));
actions.add(new OpenAutoIngestLogAction(nodeData));
- actions.add(deleteCaseInputAction);
- actions.add(deleteCaseOutputAction);
- actions.add(deleteCaseAction);
- actions.add(new ShowCaseDeletionStatusAction(nodeData));
+ if (AutoIngestDashboard.extendedFeaturesAreEnabled()) {
+ actions.add(deleteCaseInputAction);
+ actions.add(deleteCaseOutputAction);
+ actions.add(deleteCaseInputAndOutputAction);
+ } else {
+ actions.add(deleteCaseAction);
+ }
return actions;
}
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardTopComponent.form b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardTopComponent.form
index 8625b7f645..3ade741f25 100755
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardTopComponent.form
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/CasesDashboardTopComponent.form
@@ -20,8 +20,12 @@
-
-
+
+
+
+
+
+
@@ -37,7 +41,11 @@
-
+
+
+
+
+
@@ -58,5 +66,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+