From f63e84741fe931d32bbe5a083cd13242939664f9 Mon Sep 17 00:00:00 2001 From: momo Date: Thu, 23 Jul 2015 15:19:03 -0400 Subject: [PATCH 1/4] added index message to PhotoRec Module --- .../modules/photoreccarver/Bundle.properties | 7 +- .../PhotoRecCarverFileIngestModule.java | 100 ++++++++++++++---- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties index 41acbcf61d..adf417ea07 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties @@ -13,4 +13,9 @@ cannotCreateOutputDir.message=Unable to create output directory: {0} PhotoRecIngestModule.processTerminated=PhotoRec Carver ingest module was terminated due to exceeding max allowable run time when scanning PhotoRecIngestModule.moduleError=PhotoRec Carver Module Error PhotoRecIngestModule.UnableToCarve=Unable to carve file: {0} -PhotoRecIngestModule.NotEnoughDiskSpace=Not enough disk space to save unallocated file. Carving will be skipped. \ No newline at end of file +PhotoRecIngestModule.NotEnoughDiskSpace=Not enough disk space to save unallocated file. Carving will be skipped. +PhotoRecIngestModule.complete.photosRecovered=Photos Recovered\: +PhotoRecIngestModule.complete.totalCalcTime=Total Calculation Time +PhotoRecIngestModule.complete.totalLookupTime=Total Lookup Time +PhotoRecIngestModule.complete.listUsed=List Used\: +PhotoRecIngestModule.complete.photoRecResults=PhotoRec Results \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index f168d20e84..df6ca53c50 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -32,7 +32,9 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.HashMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import org.openide.modules.InstalledFileLocator; import org.openide.util.NbBundle; @@ -46,6 +48,7 @@ import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.FileIngestModuleProcessTerminator; import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -71,12 +74,30 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { private static final String LOG_FILE = "run_log.txt"; //NON-NLS private static final String TEMP_DIR_NAME = "temp"; // NON-NLS private static final Logger logger = Logger.getLogger(PhotoRecCarverFileIngestModule.class.getName()); + private static final HashMap totalsForIngestJobs = new HashMap<>(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); private static final Map pathsByJob = new ConcurrentHashMap<>(); private IngestJobContext context; private Path rootOutputDirPath; private File executableFile; private IngestServices services; + private long jobId; + private List carvedItems; + + private static class IngestJobTotals { + + private AtomicLong totalPhotosRecovered = new AtomicLong(0); + private AtomicLong totalCalctime = new AtomicLong(0); + } + + private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) { + IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId); + if (totals == null) { + totals = new PhotoRecCarverFileIngestModule.IngestJobTotals(); + totalsForIngestJobs.put(ingestJobId, totals); + } + return totals; + } /** * @inheritDoc @@ -85,6 +106,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { public void startUp(IngestJobContext context) throws IngestModule.IngestModuleException { this.context = context; this.services = IngestServices.getInstance(); + this.jobId = this.context.getJobId(); // If the global unallocated space processing setting and the module // process unallocated space only setting are not in sych, throw an @@ -99,7 +121,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { Path execName = Paths.get(PHOTOREC_DIRECTORY, PHOTOREC_EXECUTABLE); executableFile = locateExecutable(execName.toString()); - if (PhotoRecCarverFileIngestModule.refCounter.incrementAndGet(this.context.getJobId()) == 1) { + if (PhotoRecCarverFileIngestModule.refCounter.incrementAndGet(this.jobId) == 1) { try { // The first instance creates an output subdirectory with a date and time stamp DateFormat dateFormat = new SimpleDateFormat("MM-dd-yyyy-HH-mm-ss-SSSS"); // NON-NLS @@ -113,9 +135,8 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { Files.createDirectory(tempDirPath); // Save the directories for the current job. - PhotoRecCarverFileIngestModule.pathsByJob.put(this.context.getJobId(), new WorkingPaths(outputDirPath, tempDirPath)); - } - catch (SecurityException | IOException | UnsupportedOperationException ex) { + PhotoRecCarverFileIngestModule.pathsByJob.put(this.jobId, new WorkingPaths(outputDirPath, tempDirPath)); + } catch (SecurityException | IOException | UnsupportedOperationException ex) { throw new IngestModule.IngestModuleException(NbBundle.getMessage(this.getClass(), "cannotCreateOutputDir.message", ex.getLocalizedMessage())); } } @@ -130,6 +151,9 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { if (file.getType() != TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) { return IngestModule.ProcessResult.OK; } + + // Safely get a reference to the totalsForIngestJobs object + IngestJobTotals totals = getTotalsForIngestJobs(jobId); Path tempFilePath = null; try { @@ -160,7 +184,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { } // Write the file to disk. - WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.get(this.context.getJobId()); + WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.get(this.jobId); tempFilePath = Paths.get(paths.getTempDirPath().toString(), file.getName()); ContentUtils.writeToFile(file, tempFilePath.toFile()); @@ -184,7 +208,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { processAndSettings.redirectOutput(Redirect.appendTo(log)); int exitValue = ExecUtil.execute(processAndSettings, new FileIngestModuleProcessTerminator(this.context)); - + if (this.context.fileIngestIsCancelled() == true) { // if it was cancelled by the user, result is OK cleanup(outputDirPath, tempFilePath); @@ -211,21 +235,22 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { } } } - + // Now that we've cleaned up the folders and data files, parse the xml output file to add carved items into the database + long calcstart = System.currentTimeMillis(); PhotoRecCarverOutputParser parser = new PhotoRecCarverOutputParser(outputDirPath); - List theList = parser.parse(newAuditFile, id, file); - if (theList != null) { // if there were any results from carving, add the unallocated carving event to the reports list. - context.addFilesToJob(new ArrayList<>(theList)); - services.fireModuleContentEvent(new ModuleContentEvent(theList.get(0))); // fire an event to update the tree + carvedItems = parser.parse(newAuditFile, id, file); + long delta = (System.currentTimeMillis() - calcstart); + totals.totalCalctime.addAndGet(delta); + if (carvedItems != null) { // if there were any results from carving, add the unallocated carving event to the reports list. + totals.totalPhotosRecovered.addAndGet(carvedItems.size()); + context.addFilesToJob(new ArrayList<>(carvedItems)); + services.fireModuleContentEvent(new ModuleContentEvent(carvedItems.get(0))); // fire an event to update the tree } - } - catch (IOException ex) { + } catch (IOException ex) { logger.log(Level.SEVERE, "Error processing " + file.getName() + " with PhotoRec carver", ex); // NON-NLS return IngestModule.ProcessResult.ERROR; - } - - finally { + } finally { if (null != tempFilePath && Files.exists(tempFilePath)) { // Get rid of the unallocated space file. tempFilePath.toFile().delete(); @@ -234,7 +259,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { return IngestModule.ProcessResult.OK; } - + private void cleanup(Path outputDirPath, Path tempFilePath) { // cleanup the output path FileUtil.deleteDir(new File(outputDirPath.toString())); @@ -243,19 +268,54 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { } } + private synchronized void postSummary() { + IngestJobTotals jobTotals = totalsForIngestJobs.remove(jobId); + + StringBuilder detailsSb = new StringBuilder(); + //details + detailsSb.append(""); //NON-NLS + + detailsSb.append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + + detailsSb.append("\n"); //NON-NLS + detailsSb.append("
") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.photosRecovered")) + .append("").append(jobTotals.totalPhotosRecovered.get()).append("
") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.totalCalcTime")) + .append("").append(jobTotals.totalCalctime.get()).append("
"); //NON-NLS + + detailsSb.append("

") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.listUsed")) + .append("

\n
    "); //NON-NLS + for (LayoutFile lf : carvedItems) { + detailsSb.append("
  • ").append(lf.getName()).append("
    • \n"); //NON-NLS + } + + detailsSb.append("
"); //NON-NLS + + services.postMessage(IngestMessage.createMessage( + IngestMessage.MessageType.INFO, + PhotoRecCarverIngestModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "PhotoRecIngestModule.complete.photoRecResults"), + detailsSb.toString())); + + } /** * @inheritDoc */ @Override public void shutDown() { - if (this.context != null && refCounter.decrementAndGet(this.context.getJobId()) == 0) { + if (this.context != null && refCounter.decrementAndGet(this.jobId) == 0) { try { // The last instance of this module for an ingest job cleans out // the working paths map entry for the job and deletes the temp dir. - WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.remove(this.context.getJobId()); + WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.remove(this.jobId); FileUtil.deleteDir(new File(paths.getTempDirPath().toString())); - } + postSummary(); + } catch (SecurityException ex) { logger.log(Level.SEVERE, "Error shutting down PhotoRec carver module", ex); // NON-NLS } From cdbb3de0bff10b2f97db365a7768ca7c269ca919 Mon Sep 17 00:00:00 2001 From: momo Date: Thu, 23 Jul 2015 15:51:37 -0400 Subject: [PATCH 2/4] changing to more appropriate variable names and descriptions --- .../modules/photoreccarver/Bundle.properties | 8 ++--- .../PhotoRecCarverFileIngestModule.java | 29 ++++++++++++------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties index adf417ea07..2dbb6d3353 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties @@ -14,8 +14,8 @@ PhotoRecIngestModule.processTerminated=PhotoRec Carver ingest module was termina PhotoRecIngestModule.moduleError=PhotoRec Carver Module Error PhotoRecIngestModule.UnableToCarve=Unable to carve file: {0} PhotoRecIngestModule.NotEnoughDiskSpace=Not enough disk space to save unallocated file. Carving will be skipped. -PhotoRecIngestModule.complete.photosRecovered=Photos Recovered\: -PhotoRecIngestModule.complete.totalCalcTime=Total Calculation Time -PhotoRecIngestModule.complete.totalLookupTime=Total Lookup Time -PhotoRecIngestModule.complete.listUsed=List Used\: +PhotoRecIngestModule.complete.numberOfCarved=Number of Files Carved\: +PhotoRecIngestModule.complete.totalWritetime=Total Time To Write To Disk +PhotoRecIngestModule.complete.totalParsetime=Total Parsing Time +PhotoRecIngestModule.complete.recFiles=List of Recovered Files\: PhotoRecIngestModule.complete.photoRecResults=PhotoRec Results \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index df6ca53c50..255b8ad899 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -87,7 +87,8 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { private static class IngestJobTotals { private AtomicLong totalPhotosRecovered = new AtomicLong(0); - private AtomicLong totalCalctime = new AtomicLong(0); + private AtomicLong totalWritetime = new AtomicLong(0); + private AtomicLong totalParsetime = new AtomicLong(0); } private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) { @@ -184,6 +185,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { } // Write the file to disk. + long writestart = System.currentTimeMillis(); WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.get(this.jobId); tempFilePath = Paths.get(paths.getTempDirPath().toString(), file.getName()); ContentUtils.writeToFile(file, tempFilePath.toFile()); @@ -235,13 +237,15 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { } } } - + long writedelta = (System.currentTimeMillis() - writestart); + totals.totalWritetime.addAndGet(writedelta); + // Now that we've cleaned up the folders and data files, parse the xml output file to add carved items into the database long calcstart = System.currentTimeMillis(); PhotoRecCarverOutputParser parser = new PhotoRecCarverOutputParser(outputDirPath); carvedItems = parser.parse(newAuditFile, id, file); - long delta = (System.currentTimeMillis() - calcstart); - totals.totalCalctime.addAndGet(delta); + long calcdelta = (System.currentTimeMillis() - calcstart); + totals.totalParsetime.addAndGet(calcdelta); if (carvedItems != null) { // if there were any results from carving, add the unallocated carving event to the reports list. totals.totalPhotosRecovered.addAndGet(carvedItems.size()); context.addFilesToJob(new ArrayList<>(carvedItems)); @@ -276,18 +280,21 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { detailsSb.append(""); //NON-NLS detailsSb.append(""); //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.numberOfCarved")) + .append(""); //NON-NLS detailsSb.append(""); //NON-NLS detailsSb.append("\n"); //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.totalWritetime")) + .append("\n"); //NON-NLS + detailsSb.append("\n"); //NON-NLS detailsSb.append("
") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.photosRecovered")) - .append("").append(jobTotals.totalPhotosRecovered.get()).append("
") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.totalCalcTime")) - .append("").append(jobTotals.totalCalctime.get()).append("
").append(jobTotals.totalWritetime.get()).append("
") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.totalParsetime")) + .append("").append(jobTotals.totalParsetime.get()).append("
"); //NON-NLS detailsSb.append("

") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.listUsed")) - .append("

\n
    "); //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.recFiles")) + .append("

    \n
      "); //NON-NLS for (LayoutFile lf : carvedItems) { detailsSb.append("
    • ").append(lf.getName()).append("
      • \n"); //NON-NLS } @@ -298,7 +305,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { IngestMessage.MessageType.INFO, PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), - "PhotoRecIngestModule.complete.photoRecResults"), + "PhotoRecIngestModule.complete.photoRecResults"), detailsSb.toString())); } From 05b4d42b45c9b49f1bfe806f010fec54c81e213b Mon Sep 17 00:00:00 2001 From: momo Date: Thu, 23 Jul 2015 16:32:47 -0400 Subject: [PATCH 3/4] removing the list of carved files from message --- .../autopsy/modules/photoreccarver/Bundle.properties | 1 - .../PhotoRecCarverFileIngestModule.java | 12 +----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties index 2dbb6d3353..2bb4e97908 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties @@ -17,5 +17,4 @@ PhotoRecIngestModule.NotEnoughDiskSpace=Not enough disk space to save unallocate PhotoRecIngestModule.complete.numberOfCarved=Number of Files Carved\: PhotoRecIngestModule.complete.totalWritetime=Total Time To Write To Disk PhotoRecIngestModule.complete.totalParsetime=Total Parsing Time -PhotoRecIngestModule.complete.recFiles=List of Recovered Files\: PhotoRecIngestModule.complete.photoRecResults=PhotoRec Results \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index 255b8ad899..7311afc414 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -82,7 +82,6 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { private File executableFile; private IngestServices services; private long jobId; - private List carvedItems; private static class IngestJobTotals { @@ -243,7 +242,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { // Now that we've cleaned up the folders and data files, parse the xml output file to add carved items into the database long calcstart = System.currentTimeMillis(); PhotoRecCarverOutputParser parser = new PhotoRecCarverOutputParser(outputDirPath); - carvedItems = parser.parse(newAuditFile, id, file); + List carvedItems = parser.parse(newAuditFile, id, file); long calcdelta = (System.currentTimeMillis() - calcstart); totals.totalParsetime.addAndGet(calcdelta); if (carvedItems != null) { // if there were any results from carving, add the unallocated carving event to the reports list. @@ -292,15 +291,6 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { .append("").append(jobTotals.totalParsetime.get()).append("\n"); //NON-NLS detailsSb.append(""); //NON-NLS - detailsSb.append("

      ") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.recFiles")) - .append("

      \n
        "); //NON-NLS - for (LayoutFile lf : carvedItems) { - detailsSb.append("
      • ").append(lf.getName()).append("
        • \n"); //NON-NLS - } - - detailsSb.append("
      "); //NON-NLS - services.postMessage(IngestMessage.createMessage( IngestMessage.MessageType.INFO, PhotoRecCarverIngestModuleFactory.getModuleName(), From ec0cc450c09c5a565e56e6fcf362679b1b47b7a2 Mon Sep 17 00:00:00 2001 From: momo Date: Mon, 27 Jul 2015 09:42:35 -0400 Subject: [PATCH 4/4] changed variable name to represent more than just photos --- .../photoreccarver/PhotoRecCarverFileIngestModule.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index 7311afc414..e0277e524a 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -85,7 +85,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { private static class IngestJobTotals { - private AtomicLong totalPhotosRecovered = new AtomicLong(0); + private AtomicLong totalItemsRecovered = new AtomicLong(0); private AtomicLong totalWritetime = new AtomicLong(0); private AtomicLong totalParsetime = new AtomicLong(0); } @@ -246,7 +246,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { long calcdelta = (System.currentTimeMillis() - calcstart); totals.totalParsetime.addAndGet(calcdelta); if (carvedItems != null) { // if there were any results from carving, add the unallocated carving event to the reports list. - totals.totalPhotosRecovered.addAndGet(carvedItems.size()); + totals.totalItemsRecovered.addAndGet(carvedItems.size()); context.addFilesToJob(new ArrayList<>(carvedItems)); services.fireModuleContentEvent(new ModuleContentEvent(carvedItems.get(0))); // fire an event to update the tree } @@ -281,7 +281,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { detailsSb.append("") //NON-NLS .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.numberOfCarved")) .append(""); //NON-NLS - detailsSb.append("").append(jobTotals.totalPhotosRecovered.get()).append(""); //NON-NLS + detailsSb.append("").append(jobTotals.totalItemsRecovered.get()).append(""); //NON-NLS detailsSb.append("") //NON-NLS .append(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.complete.totalWritetime"))