From bd480827b509d661921589c48e83e38c9ee34369 Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 16 Jul 2020 13:24:25 -0400 Subject: [PATCH 01/24] Use streaming ingest for disk images in auto ingest --- .../casemodule/DefaultIngestStream.java | 6 ++++ .../autopsy/casemodule/ImageDSProcessor.java | 36 +++++++++++++++++++ .../AutoIngestDataSourceProcessor.java | 6 ++++ .../autopsy/ingest/IngestJobInputStream.java | 5 +++ .../autopsy/ingest/IngestStream.java | 2 ++ .../autoingest/AutoIngestManager.java | 27 +++++++++++--- 6 files changed, 77 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java index 08bc0fa427..dd83f18694 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.casemodule; import java.util.List; +import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestStream; import org.sleuthkit.autopsy.ingest.IngestStreamClosedException; @@ -35,6 +36,11 @@ class DefaultIngestStream implements IngestStream { public void addFiles(List fileObjectIds) throws IngestStreamClosedException { // Do nothing } + + @Override + public IngestJob getIngestJob() { + return null; + } @Override public synchronized boolean isClosed() { diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java index 9dd9a39fd4..207b83ed8a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java @@ -464,6 +464,42 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); } + + @Override + public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + this.deviceId = deviceId; + this.imagePath = dataSourcePath.toString(); + this.sectorSize = 0; + this.timeZone = Calendar.getInstance().getTimeZone().getID(); + this.ignoreFatOrphanFiles = false; + setDataSourceOptionsCalled = true; + + // Set up the data source before creating the ingest stream + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return null; + } + + // Now initialize the ingest stream + try { + ingestStream = IngestManager.getInstance().openIngestStream(image, settings); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error starting ingest modules", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return null; + } + + doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); + return ingestStream; + } /** * Sets the configuration of the data source processor without using the diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java index 20f3bb59bd..95956bb5b0 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java @@ -22,6 +22,8 @@ import java.nio.file.Path; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.ingest.IngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestStream; /** * Interface implemented by DataSourceProcessors in order to be supported by @@ -66,6 +68,10 @@ public interface AutoIngestDataSourceProcessor extends DataSourceProcessor { */ void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack); + default IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor"); + } + /** * A custom exception for the use of AutomatedIngestDataSourceProcessor. */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java index 4e8e9c4019..a2687d5c1d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java @@ -58,6 +58,11 @@ class IngestJobInputStream implements IngestStream { } ingestJob.addStreamingIngestFiles(fileObjectIds); } + + @Override + public IngestJob getIngestJob() { + return ingestJob; + } @Override public synchronized void close() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java index 62a42af208..4776056c98 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java @@ -34,6 +34,8 @@ public interface IngestStream { * @throws IngestStreamClosedException */ void addFiles(List fileObjectIds) throws IngestStreamClosedException; + + IngestJob getIngestJob(); /** * Closes the ingest stream. Should be called after all files from data diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 48b86025af..da86ff8263 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -101,6 +101,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; +import org.sleuthkit.autopsy.ingest.IngestStream; import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException; import org.sleuthkit.autopsy.keywordsearch.Server; import org.sleuthkit.datamodel.Content; @@ -165,6 +166,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen private AutoIngestJob currentJob; @GuardedBy("jobsLock") private List completedJobs; + private IngestStream currentIngestStream = null; private CoordinationService coordinationService; private JobProcessingTask jobProcessingTask; private Future jobProcessingTaskFuture; @@ -2443,6 +2445,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen return; } + currentIngestStream = null; runDataSourceProcessor(caseForJob, dataSource); if (dataSource.getContent().isEmpty()) { currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now())); @@ -2558,7 +2561,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen caseForJob.notifyAddingDataSource(taskId); jobLogger.logDataSourceProcessorSelected(selectedProcessor.getDataSourceType()); sysLogger.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()}); - selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + if (selectedProcessor.supportsIngestStream()) { + IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); + // TODO check for settings errors + currentIngestStream = selectedProcessor.processWithIngestStream(dataSource.getDeviceId(), dataSource.getPath(), ingestJobSettings, progressMonitor, callBack); + } else { + selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + } ingestLock.wait(); // at this point we got the content object(s) from the current DSP. @@ -2674,11 +2683,19 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { + // TODO don't do all this IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); List settingsWarnings = ingestJobSettings.getWarnings(); if (settingsWarnings.isEmpty()) { - IngestJobStartResult ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); - IngestJob ingestJob = ingestJobStartResult.getJob(); + + IngestJobStartResult ingestJobStartResult = null; + IngestJob ingestJob; + if (currentIngestStream == null) { + ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); + ingestJob = ingestJobStartResult.getJob(); + } else { + ingestJob = currentIngestStream.getIngestJob(); + } if (null != ingestJob) { currentJob.setIngestJob(ingestJob); /* @@ -2714,7 +2731,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen } } } - } else if (!ingestJobStartResult.getModuleErrors().isEmpty()) { + } else if (ingestJobStartResult != null && !ingestJobStartResult.getModuleErrors().isEmpty()) { for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) { sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); } @@ -2722,7 +2739,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen setErrorsOccurredFlagForCase(caseDirectoryPath); jobLogger.logIngestModuleStartupErrors(); throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); - } else { + } else if (ingestJobStartResult != null) { sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); currentJob.setErrorsOccurred(true); setErrorsOccurredFlagForCase(caseDirectoryPath); From 080847893e7c4cee3ba02bfab8fb96e259632c44 Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 16 Jul 2020 14:16:48 -0400 Subject: [PATCH 02/24] Improve TODOs --- .../experimental/autoingest/AutoIngestManager.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index da86ff8263..a3810d290f 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2563,7 +2563,16 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen sysLogger.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()}); if (selectedProcessor.supportsIngestStream()) { IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); - // TODO check for settings errors + if (! ingestJobSettings.getWarnings().isEmpty()) { + for (String warning : ingestJobSettings.getWarnings()) { + sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); + } + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logIngestJobSettingsErrors(); + // TODO Change exception type + throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Error(s) in ingest job settings"); + } currentIngestStream = selectedProcessor.processWithIngestStream(dataSource.getDeviceId(), dataSource.getPath(), ingestJobSettings, progressMonitor, callBack); } else { selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); @@ -2683,7 +2692,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { - // TODO don't do all this + // TODO Don't redo loading the settings when there's an ingest stream IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); List settingsWarnings = ingestJobSettings.getWarnings(); if (settingsWarnings.isEmpty()) { From 24ed23b3b0f18090c4da522c7d381b1b759228ca Mon Sep 17 00:00:00 2001 From: apriestman Date: Mon, 3 Aug 2020 13:42:44 -0400 Subject: [PATCH 03/24] Cleanup/commenting --- .../casemodule/DefaultIngestStream.java | 2 +- .../AutoIngestDataSourceProcessor.java | 21 +++++++++++++++++++ .../autopsy/ingest/IngestStream.java | 5 +++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java index dd83f18694..98f2855fcb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java @@ -39,7 +39,7 @@ class DefaultIngestStream implements IngestStream { @Override public IngestJob getIngestJob() { - return null; + throw new UnsupportedOperationException("DefaultIngestStream has no associated IngestJob"); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java index 95956bb5b0..27ffec6d53 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java @@ -68,6 +68,27 @@ public interface AutoIngestDataSourceProcessor extends DataSourceProcessor { */ void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack); + + /** + * Adds a data source to the case database using a background task in a + * separate thread by calling DataSourceProcessor.run() method. Returns as + * soon as the background task is started. The background task uses a + * callback object to signal task completion and return results. Method can + * throw an exception for a system level problem. The exception should not + * be thrown for an issue related to bad input data. + * + * @param deviceId An ASCII-printable identifier for the device + * associated with the data source that is intended + * to be unique across multiple cases (e.g., a UUID). + * @param dataSourcePath Path to the data source. + * @param settings The ingest job settings. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callBack Callback that will be used by the background task + * to return results. + * + * @return The new ingest stream or null if an error occurred. Errors will be handled by the callback. + */ default IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor"); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java index 4776056c98..77001531be 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java @@ -35,6 +35,11 @@ public interface IngestStream { */ void addFiles(List fileObjectIds) throws IngestStreamClosedException; + /** + * Get the ingest job associated with this ingest stream. + * + * @return The IngestJob. + */ IngestJob getIngestJob(); /** From a99c225f0bfbecaafc0ad50212ad6beff14d031e Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 6 Aug 2020 14:11:23 -0400 Subject: [PATCH 04/24] Cleanup --- .../autoingest/AutoIngestJobLogger.java | 15 ++ .../autoingest/AutoIngestManager.java | 133 +++++++++--------- 2 files changed, 85 insertions(+), 63 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java index 708ce7ee14..5bc3f46fac 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java @@ -275,6 +275,21 @@ final class AutoIngestJobLogger { void logIngestJobSettingsErrors() throws AutoIngestJobLoggerException, InterruptedException { log(MessageCategory.ERROR, "Failed to analyze data source due to settings errors"); } + + /** + * Logs failure to analyze a data source, possibly due to ingest job settings errors. + * Used with streaming ingest since incorrect settings are the most likely cause + * of the error. + * + * @throws AutoIngestJobLoggerException if there is an error writing the log + * message. + * @throws InterruptedException if interrupted while blocked waiting + * to acquire an exclusive lock on the + * log file. + */ + void logProbableIngestJobSettingsErrors() throws AutoIngestJobLoggerException, InterruptedException { + log(MessageCategory.ERROR, "Failed to analyze data source, probably due to ingest settings errors"); + } /** * Logs failure to analyze a data source due to ingest module startup diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index a3810d290f..6e1fb7f610 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2570,10 +2570,17 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen currentJob.setErrorsOccurred(true); setErrorsOccurredFlagForCase(caseDirectoryPath); jobLogger.logIngestJobSettingsErrors(); - // TODO Change exception type - throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Error(s) in ingest job settings"); + throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Error(s) in ingest job settings for " + manifestPath); } currentIngestStream = selectedProcessor.processWithIngestStream(dataSource.getDeviceId(), dataSource.getPath(), ingestJobSettings, progressMonitor, callBack); + if (currentIngestStream == null) { + // Either there was a failure to add the data source object to the database or the ingest settings were bad. + // An error in the ingest settings is the more likely scenario. + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logProbableIngestJobSettingsErrors(); + throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Error initializing processing for " + manifestPath + ", probably due to an ingest settings error"); + } } else { selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); } @@ -2692,77 +2699,77 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { - // TODO Don't redo loading the settings when there's an ingest stream - IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); - List settingsWarnings = ingestJobSettings.getWarnings(); - if (settingsWarnings.isEmpty()) { - - IngestJobStartResult ingestJobStartResult = null; - IngestJob ingestJob; - if (currentIngestStream == null) { - ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); - ingestJob = ingestJobStartResult.getJob(); - } else { - ingestJob = currentIngestStream.getIngestJob(); + IngestJob ingestJob; + IngestJobStartResult ingestJobStartResult = null; + if (currentIngestStream == null) { + IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); + List settingsWarnings = ingestJobSettings.getWarnings(); + if (! settingsWarnings.isEmpty()) { + for (String warning : settingsWarnings) { + sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); + } + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logIngestJobSettingsErrors(); + throw new AnalysisStartupException("Error(s) in ingest job settings"); } - if (null != ingestJob) { - currentJob.setIngestJob(ingestJob); - /* - * Block until notified by the ingest job event - * listener or until interrupted because auto ingest - * is shutting down. - */ - ingestLock.wait(); - sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); - IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); - for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { - AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); - if (!snapshot.isCancelled()) { - List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); - if (!cancelledModules.isEmpty()) { - sysLogger.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath)); - currentJob.setErrorsOccurred(true); - setErrorsOccurredFlagForCase(caseDirectoryPath); - for (String module : snapshot.getCancelledDataSourceIngestModules()) { - sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); - nestedJobLogger.logIngestModuleCancelled(module); - } - } - nestedJobLogger.logAnalysisCompleted(); - } else { - currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); + + + ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); + ingestJob = ingestJobStartResult.getJob(); + } else { + ingestJob = currentIngestStream.getIngestJob(); + } + + if (null != ingestJob) { + currentJob.setIngestJob(ingestJob); + /* + * Block until notified by the ingest job event + * listener or until interrupted because auto ingest + * is shutting down. + */ + ingestLock.wait(); + sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); + IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); + for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { + AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); + if (!snapshot.isCancelled()) { + List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); + if (!cancelledModules.isEmpty()) { + sysLogger.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath)); currentJob.setErrorsOccurred(true); setErrorsOccurredFlagForCase(caseDirectoryPath); - nestedJobLogger.logAnalysisCancelled(); - CancellationReason cancellationReason = snapshot.getCancellationReason(); - if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { - throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); + for (String module : snapshot.getCancelledDataSourceIngestModules()) { + sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); + nestedJobLogger.logIngestModuleCancelled(module); } } + nestedJobLogger.logAnalysisCompleted(); + } else { + currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + nestedJobLogger.logAnalysisCancelled(); + CancellationReason cancellationReason = snapshot.getCancellationReason(); + if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { + throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); + } } - } else if (ingestJobStartResult != null && !ingestJobStartResult.getModuleErrors().isEmpty()) { - for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) { - sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); - } - currentJob.setErrorsOccurred(true); - setErrorsOccurredFlagForCase(caseDirectoryPath); - jobLogger.logIngestModuleStartupErrors(); - throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); - } else if (ingestJobStartResult != null) { - sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); - currentJob.setErrorsOccurred(true); - setErrorsOccurredFlagForCase(caseDirectoryPath); - jobLogger.logAnalysisStartupError(); - throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException()); } - } else { - for (String warning : settingsWarnings) { - sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); + } else if (ingestJobStartResult != null && !ingestJobStartResult.getModuleErrors().isEmpty()) { + for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) { + sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); } currentJob.setErrorsOccurred(true); setErrorsOccurredFlagForCase(caseDirectoryPath); - jobLogger.logIngestJobSettingsErrors(); - throw new AnalysisStartupException("Error(s) in ingest job settings"); + jobLogger.logIngestModuleStartupErrors(); + throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); + } else if (ingestJobStartResult != null) { + sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logAnalysisStartupError(); + throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException()); } } } finally { From 85e459601444927c70f81223fdca28225a35e7d5 Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 6 Aug 2020 14:26:00 -0400 Subject: [PATCH 05/24] Cancel ingest if the DSP fails --- .../autopsy/experimental/autoingest/AutoIngestManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 6e1fb7f610..7c817e7739 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2593,6 +2593,12 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen // move onto the the next DSP that can process this data source jobLogger.logDataSourceProcessorError(selectedProcessor.getDataSourceType()); logDataSourceProcessorResult(dataSource); + + // If we had created an ingest stream, close it + if (currentIngestStream != null) { + currentIngestStream.stop(); + currentIngestStream = null; + } continue; } From 5320ca01c28eebd20b2430a561cf19594233849f Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 3 Sep 2020 15:43:50 -0400 Subject: [PATCH 06/24] Test multiple matching search engines. General changes to use existing APIs. --- .../autopsy/recentactivity/SEUQAMappings.xml | 52 ++++----- .../SearchEngineURLQueryAnalyzer.java | 103 ++++++++++-------- 2 files changed, 85 insertions(+), 70 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml index bbc4d12f61..098041fb63 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml @@ -12,7 +12,7 @@ Each splitToken contains a single mapping of a raw URL substring to its regex eq SearchEngine: engine: The engines basic name - domainSubstring: The domain of the URL such that it can uniquely be identified as given engine. + domainSubstring: The domain of the URL such that it can uniquely be identified as given engine. Should not have leading or trailing '.' splitToken: plainToken: The string in the URL that is immediately followed by the actual query. @@ -25,30 +25,30 @@ splitToken: --> - + - + - + - + - + - + - + @@ -59,28 +59,28 @@ splitToken: - + - + - + - + - + @@ -92,22 +92,22 @@ splitToken: - + - + - + - + - + @@ -116,28 +116,28 @@ splitToken: - + - + - + - + - + - + - + - + diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index eeb6e5a987..27e50c19e1 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -22,10 +22,15 @@ import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.util.Arrays; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -107,11 +112,13 @@ class SearchEngineURLQueryAnalyzer extends Extract { private final String engineName; private final String domainSubstring; private final List keyPairs; + private final Pattern domainRegexPattern; private int count; SearchEngine(String engineName, String domainSubstring, List keyPairs) { this.engineName = engineName; this.domainSubstring = domainSubstring; + domainRegexPattern = Pattern.compile("^(.*[./])?" + domainSubstring + "([./].*)?$"); this.keyPairs = keyPairs; count = 0; } @@ -127,6 +134,10 @@ class SearchEngineURLQueryAnalyzer extends Extract { String getDomainSubstring() { return domainSubstring; } + + Pattern getDomainRegexPattern() { + return domainRegexPattern; + } int getTotal() { return count; @@ -202,20 +213,21 @@ class SearchEngineURLQueryAnalyzer extends Extract { * * @param domain domain as part of the URL * - * @return supported search engine the domain belongs to or null if no match - * is found + * @return supported search engine(s) the domain belongs to (list may be empty) * */ - private static SearchEngineURLQueryAnalyzer.SearchEngine getSearchEngineFromUrl(String domain) { + private static Collection getSearchEngineFromUrl(String domain) { + List supportedEngines = new ArrayList<>(); if (engines == null) { - return null; + return supportedEngines; } for (SearchEngine engine : engines) { - if (domain.contains(engine.getDomainSubstring())) { - return engine; + Matcher matcher = engine.getDomainRegexPattern().matcher(domain); + if (matcher.matches()) { + supportedEngines.add(engine); } } - return null; + return supportedEngines; } /** @@ -294,8 +306,9 @@ class SearchEngineURLQueryAnalyzer extends Extract { int totalQueries = 0; try { //from blackboard_artifacts - Collection listArtifacts = currentCase.getSleuthkitCase().getMatchingArtifacts("WHERE (blackboard_artifacts.artifact_type_id = '" + ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID() //NON-NLS - + "' OR blackboard_artifacts.artifact_type_id = '" + ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID() + "') "); //List of every 'web_history' and 'bookmark' artifact NON-NLS + Collection listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts( + Arrays.asList(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_BOOKMARK), new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY)), + Arrays.asList(dataSource.getId())); logger.log(Level.INFO, "Processing {0} blackboard artifacts.", listArtifacts.size()); //NON-NLS for (BlackboardArtifact artifact : listArtifacts) { @@ -304,51 +317,54 @@ class SearchEngineURLQueryAnalyzer extends Extract { } //initializing default attributes - String query = ""; String searchEngineDomain = ""; String browser = ""; long last_accessed = -1; - long fileId = artifact.getObjectID(); - boolean isFromSource = tskCase.isFileFromSource(dataSource, fileId); - if (!isFromSource) { - //File was from a different dataSource. Skipping. - continue; - } - - AbstractFile file = tskCase.getAbstractFileById(fileId); + AbstractFile file = tskCase.getAbstractFileById(artifact.getObjectID()); if (file == null) { continue; } - SearchEngineURLQueryAnalyzer.SearchEngine se = null; - //from blackboard_attributes - Collection listAttributes = currentCase.getSleuthkitCase().getMatchingAttributes("WHERE artifact_id = " + artifact.getArtifactID()); //NON-NLS - - for (BlackboardAttribute attribute : listAttributes) { - if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL.getTypeID()) { - final String urlString = attribute.getValueString(); - se = getSearchEngineFromUrl(urlString); - if (se == null) { - break; - } - - query = extractSearchEngineQuery(se, attribute.getValueString()); - if (query.equals("")) //False positive match, artifact was not a query. NON-NLS - { - break; - } - - } else if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID()) { - browser = attribute.getValueString(); - } else if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID()) { - searchEngineDomain = attribute.getValueString(); - } else if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID()) { - last_accessed = attribute.getValueLong(); + // Try search engines on the URL to see if any produce a search string + Set searchQueries = new HashSet<>(); + BlackboardAttribute urlAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL)); + if (urlAttr == null) { + continue; + } + + final String urlString = urlAttr.getValueString(); + Collection possibleSearchEngines = getSearchEngineFromUrl(urlString); + for (SearchEngineURLQueryAnalyzer.SearchEngine se : possibleSearchEngines) { + String query = extractSearchEngineQuery(se, urlString); + // If we have a non-empty query string, add it to the list + if ( !query.equals("")) { + searchQueries.add(query); + se.increment(); } } + + // If we didn't extract any search queries, go on to the next artifact + if (searchQueries.isEmpty()) { + continue; + } + + // Extract the rest of the fields needed for the web search artifact + BlackboardAttribute browserAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME)); + if (browserAttr != null) { + browser = browserAttr.getValueString(); + } + BlackboardAttribute domainAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN)); + if (domainAttr != null) { + searchEngineDomain = domainAttr.getValueString(); + } + BlackboardAttribute lastAccessAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED)); + if (lastAccessAttr != null) { + last_accessed = lastAccessAttr.getValueLong(); + } - if (se != null && !query.equals("")) { //NON-NLS + // Make an artifact for each distinct query + for (String query : searchQueries) { // If date doesn't exist, change to 0 (instead of 1969) if (last_accessed == -1) { last_accessed = 0; @@ -367,7 +383,6 @@ class SearchEngineURLQueryAnalyzer extends Extract { NbBundle.getMessage(this.getClass(), "SearchEngineURLQueryAnalyzer.parentModuleName"), last_accessed)); postArtifact(createArtifactWithAttributes(ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY, file, bbattributes)); - se.increment(); ++totalQueries; } } From af304223db4f536af74494ac72ed9e513068ec81 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 4 Sep 2020 10:17:59 -0400 Subject: [PATCH 07/24] datamodel service --- .../datamodel/DataSourceInfoUtilities.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java index 9a60bd17e7..da02cb087b 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java @@ -35,6 +35,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; +import org.sleuthkit.datamodel.BlackboardAttribute.Type; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.TskData.TSK_FS_META_FLAG_ENUM; @@ -357,4 +358,36 @@ final class DataSourceInfoUtilities { } } } + + + /** + * Retrieves attribute from artifact if exists. Returns null if attribute is + * null or underlying call throws exception. + * + * @param artifact The artifact. + * @param attributeType The attribute type to retrieve from the artifact. + * + * @return The attribute or null if could not be received. + */ + private static BlackboardAttribute getAttributeOrNull(BlackboardArtifact artifact, Type attributeType) { + try { + return artifact.getAttribute(attributeType); + } catch (TskCoreException ex) { + return null; + } + } + + /** + * Retrieves the string value of a certain attribute type from an artifact. + * + * @param artifact The artifact. + * @param attributeType The attribute type. + * + * @return The 'getValueString()' value or null if the attribute or String + * could not be retrieved. + */ + static String getStringOrNull(BlackboardArtifact artifact, Type attributeType) { + BlackboardAttribute attr = getAttributeOrNull(artifact, attributeType); + return (attr == null) ? null : attr.getValueString(); + } } From 837f4a47ec5ab7b51973af865230e8b053d38382 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 4 Sep 2020 10:18:10 -0400 Subject: [PATCH 08/24] datamodel service --- .../datamodel/DataSourceAnalysisSummary.java | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java new file mode 100644 index 0000000000..a42a46b56e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java @@ -0,0 +1,89 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourcesummary.datamodel; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Providing data for the data source analysis tab. + */ +public class DataSourceAnalysisSummary { + private static final BlackboardAttribute.Type TYPE_SET_NAME = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_SET_NAME); + + private final java.util.logging.Logger logger; + private final SleuthkitCaseProvider provider; + + public DataSourceAnalysisSummary() { + this(Logger.getLogger(DataSourceAnalysisSummary.class.getName()), SleuthkitCaseProvider.DEFAULT); + } + + public DataSourceAnalysisSummary(java.util.logging.Logger logger, SleuthkitCaseProvider provider) { + this.logger = logger; + this.provider = provider; + } + + public List> getHashsetCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { + return getCountsData(dataSource, TYPE_SET_NAME, ARTIFACT_TYPE.TSK_HASHSET_HIT); + } + + public List> getKeywordCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { + return getCountsData(dataSource, TYPE_SET_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT); + } + + public List> getInterestingItemCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { + return getCountsData(dataSource, TYPE_SET_NAME, ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT); + } + + private List> getCountsData(DataSource dataSource, BlackboardAttribute.Type keyType, ARTIFACT_TYPE... artifactTypes) throws SleuthkitCaseProviderException, TskCoreException { + List artifacts = new ArrayList<>(); + SleuthkitCase skCase = provider.get(); + + for (ARTIFACT_TYPE type : artifactTypes) { + artifacts.addAll(skCase.getBlackboard().getArtifacts(type.getTypeID(), dataSource.getId())); + } + + Map countedKeys = artifacts.stream() + .map((art) -> { + String key = DataSourceInfoUtilities.getStringOrNull(art, keyType); + return (StringUtils.isBlank(key)) ? null : key; + }) + .filter((key) -> key != null) + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + + return countedKeys.entrySet().stream() + .map((e) -> Pair.of(e.getKey(), e.getValue())) + .sorted((a,b) -> -a.getValue().compareTo(b.getValue())) + .collect(Collectors.toList()); + } +} From 0114a3430c7242b9b6524ac65c7d5a1b337e791e Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Fri, 4 Sep 2020 16:06:35 -0400 Subject: [PATCH 09/24] Modified discover group panel label renderer code --- .../autopsy/discovery/GroupListPanel.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java index a71876e88d..6f3d2503d6 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java @@ -20,8 +20,11 @@ package org.sleuthkit.autopsy.discovery; import com.google.common.eventbus.Subscribe; import java.awt.Cursor; +import java.awt.Graphics2D; +import java.awt.font.FontRenderContext; import java.util.List; import java.util.Map; +import java.awt.geom.Rectangle2D; import javax.swing.DefaultListCellRenderer; import javax.swing.DefaultListModel; import javax.swing.JList; @@ -203,12 +206,32 @@ final class GroupListPanel extends javax.swing.JPanel { Object newValue = value; if (newValue instanceof GroupKey) { String valueString = newValue.toString(); - setToolTipText(valueString); - //if paths would be longer than 37 characters shorten them to be 37 characters - if (groupingAttribute instanceof FileSearch.ParentPathAttribute && valueString.length() > 37) { - valueString = valueString.substring(0, 16) + " ... " + valueString.substring(valueString.length() - 16); + setToolTipText(valueString); + + valueString += " (" + groupMap.get(newValue) + ")"; + + if (groupingAttribute instanceof FileSearch.ParentPathAttribute) { + // Using the list FontRenderContext instead of this because + // the label RenderContext was sometimes null, but this should work. + FontRenderContext context = ((Graphics2D)list.getGraphics()).getFontRenderContext(); + + //Determine the width of the string with the given font. + double stringWidth = getFont().getStringBounds(valueString, context).getWidth(); + // subtracting 10 from the width as a littl inset. + int listWidth = list.getWidth() - 10; + + if(stringWidth > listWidth ) { + double avgCharWidth = Math.floor(stringWidth/valueString.length()); + + // The extra 5 is to account for the " ... " that is being added back. + int charToRemove = (int)Math.ceil((stringWidth - listWidth) / avgCharWidth) + 5; + int charactersToShow = (int)Math.ceil((valueString.length() - charToRemove)/2); + valueString = valueString.substring(0, charactersToShow) + " ... " + valueString.substring(valueString.length() - charactersToShow); + } + + } - newValue = valueString + " (" + groupMap.get(newValue) + ")"; + newValue = valueString; } super.getListCellRendererComponent(list, newValue, index, isSelected, cellHasFocus); return this; From c43e8a988db1c4463b33431022727112528df06e Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Fri, 4 Sep 2020 16:07:39 -0400 Subject: [PATCH 10/24] Removed unused import and formatted --- .../autopsy/discovery/GroupListPanel.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java index 6f3d2503d6..3617ce3670 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java @@ -24,7 +24,6 @@ import java.awt.Graphics2D; import java.awt.font.FontRenderContext; import java.util.List; import java.util.Map; -import java.awt.geom.Rectangle2D; import javax.swing.DefaultListCellRenderer; import javax.swing.DefaultListModel; import javax.swing.JList; @@ -206,30 +205,29 @@ final class GroupListPanel extends javax.swing.JPanel { Object newValue = value; if (newValue instanceof GroupKey) { String valueString = newValue.toString(); - setToolTipText(valueString); - + setToolTipText(valueString); + valueString += " (" + groupMap.get(newValue) + ")"; - + if (groupingAttribute instanceof FileSearch.ParentPathAttribute) { // Using the list FontRenderContext instead of this because // the label RenderContext was sometimes null, but this should work. - FontRenderContext context = ((Graphics2D)list.getGraphics()).getFontRenderContext(); - + FontRenderContext context = ((Graphics2D) list.getGraphics()).getFontRenderContext(); + //Determine the width of the string with the given font. double stringWidth = getFont().getStringBounds(valueString, context).getWidth(); // subtracting 10 from the width as a littl inset. int listWidth = list.getWidth() - 10; - if(stringWidth > listWidth ) { - double avgCharWidth = Math.floor(stringWidth/valueString.length()); - + if (stringWidth > listWidth) { + double avgCharWidth = Math.floor(stringWidth / valueString.length()); + // The extra 5 is to account for the " ... " that is being added back. - int charToRemove = (int)Math.ceil((stringWidth - listWidth) / avgCharWidth) + 5; - int charactersToShow = (int)Math.ceil((valueString.length() - charToRemove)/2); + int charToRemove = (int) Math.ceil((stringWidth - listWidth) / avgCharWidth) + 5; + int charactersToShow = (int) Math.ceil((valueString.length() - charToRemove) / 2); valueString = valueString.substring(0, charactersToShow) + " ... " + valueString.substring(valueString.length() - charactersToShow); } - - + } newValue = valueString; } From 5c31586f461b80ae7c81c9c2faf2e6db2a7b1954 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Sep 2020 08:35:29 -0400 Subject: [PATCH 11/24] created analysis summary tab --- .../sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties index 350daeb57e..26c7055a75 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties @@ -32,3 +32,6 @@ DataSourceSummaryCountsPanel.byCategoryLabel.text=Files by Category DataSourceSummaryCountsPanel.resultsByTypeLabel.text=Results by Type DataSourceSummaryUserActivityPanel.programsRunLabel.text=Recent Programs DataSourceSummaryUserActivityPanel.recentDomainsLabel.text=Recent Domains +AnalysisPanel.hashsetHitsLabel.text=Hashset Hits +AnalysisPanel.keywordHitsLabel.text=Keyword Hits +AnalysisPanel.interestingItemLabel.text=Interesting Item Hits From 122b51f9f1a824a392130128068ee84809a93d35 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Sep 2020 08:35:36 -0400 Subject: [PATCH 12/24] created analysis summary tab --- .../datasourcesummary/ui/AnalysisPanel.form | 260 ++++++++++++++++++ .../datasourcesummary/ui/AnalysisPanel.java | 200 ++++++++++++++ 2 files changed, 460 insertions(+) create mode 100644 Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form create mode 100644 Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form new file mode 100644 index 0000000000..dd8ae51623 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form @@ -0,0 +1,260 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java new file mode 100644 index 0000000000..f1b1483deb --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java @@ -0,0 +1,200 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourcesummary.ui; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.DataSourceAnalysisSummary; +import org.sleuthkit.autopsy.datasourcesummary.uiutils.CellModelTableCellRenderer.DefaultCellModel; +import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchResult; +import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchWorker; +import org.sleuthkit.autopsy.datasourcesummary.uiutils.JTablePanel; +import org.sleuthkit.autopsy.datasourcesummary.uiutils.JTablePanel.ColumnModel; +import org.sleuthkit.datamodel.DataSource; + +/** + * A tab shown in data source summary displaying hash set hits, keyword hits, + * and interesting item hits within a datasource. + */ +@Messages({ + "AnalysisPanel_keyColumn_title=Name", + "AnalysisPanel_countColumn_title=Count" +}) +public class AnalysisPanel extends BaseDataSourceSummaryPanel { + + private static final long serialVersionUID = 1L; + + private static final ColumnModel> KEY_COL = new ColumnModel<>( + Bundle.AnalysisPanel_keyColumn_title(), + (pair) -> new DefaultCellModel(pair.getKey()), + 300 + ); + + private static final ColumnModel> COUNT_COL = new ColumnModel<>( + Bundle.AnalysisPanel_countColumn_title(), + (pair) -> new DefaultCellModel(String.valueOf(pair.getValue())), + 100 + ); + + private static final List>> DEFAULT_COLUMNS = Arrays.asList(KEY_COL, COUNT_COL); + + private final JTablePanel> hashsetHitsTable = JTablePanel.getJTablePanel(DEFAULT_COLUMNS); + + private final JTablePanel> keywordHitsTable = JTablePanel.getJTablePanel(DEFAULT_COLUMNS); + + private final JTablePanel> interestingItemsTable = JTablePanel.getJTablePanel(DEFAULT_COLUMNS); + + private final List> tables = Arrays.asList( + hashsetHitsTable, + keywordHitsTable, + interestingItemsTable + ); + + private final List> dataFetchComponents; + + /** + * Creates a new DataSourceUserActivityPanel. + */ + public AnalysisPanel() { + this(new DataSourceAnalysisSummary()); + } + + public AnalysisPanel(DataSourceAnalysisSummary analysisData) { + // set up data acquisition methods + dataFetchComponents = Arrays.asList( + // hashset hits loading components + new DataFetchWorker.DataFetchComponents<>( + (dataSource) -> analysisData.getHashsetCounts(dataSource), + (result) -> hashsetHitsTable.showDataFetchResult(result)), + // keyword hits loading components + new DataFetchWorker.DataFetchComponents<>( + (dataSource) -> analysisData.getKeywordCounts(dataSource), + (result) -> keywordHitsTable.showDataFetchResult(result)), + // interesting item hits loading components + new DataFetchWorker.DataFetchComponents<>( + (dataSource) -> analysisData.getInterestingItemCounts(dataSource), + (result) -> interestingItemsTable.showDataFetchResult(result)) + ); + + initComponents(); + } + + @Override + protected void onNewDataSource(DataSource dataSource) { + // if no data source is present or the case is not open, + // set results for tables to null. + if (dataSource == null || !Case.isCaseOpen()) { + this.dataFetchComponents.forEach((item) -> item.getResultHandler() + .accept(DataFetchResult.getSuccessResult(null))); + + } else { + // set tables to display loading screen + this.tables.forEach((table) -> table.showDefaultLoadingMessage()); + + // create swing workers to run for each table + List> workers = dataFetchComponents + .stream() + .map((components) -> new DataFetchWorker<>(components, dataSource)) + .collect(Collectors.toList()); + + // submit swing workers to run + submit(workers); + } + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + javax.swing.JScrollPane mainScrollPane = new javax.swing.JScrollPane(); + javax.swing.JPanel mainContentPanel = new javax.swing.JPanel(); + javax.swing.JLabel hashsetHitsLabel = new javax.swing.JLabel(); + javax.swing.Box.Filler filler1 = new javax.swing.Box.Filler(new java.awt.Dimension(0, 2), new java.awt.Dimension(0, 2), new java.awt.Dimension(32767, 2)); + javax.swing.JPanel hashSetHitsPanel = hashsetHitsTable; + javax.swing.Box.Filler filler2 = new javax.swing.Box.Filler(new java.awt.Dimension(0, 20), new java.awt.Dimension(0, 20), new java.awt.Dimension(32767, 20)); + javax.swing.JLabel keywordHitsLabel = new javax.swing.JLabel(); + javax.swing.Box.Filler filler4 = new javax.swing.Box.Filler(new java.awt.Dimension(0, 2), new java.awt.Dimension(0, 2), new java.awt.Dimension(32767, 2)); + javax.swing.JPanel keywordHitsPanel = keywordHitsTable; + javax.swing.Box.Filler filler5 = new javax.swing.Box.Filler(new java.awt.Dimension(0, 20), new java.awt.Dimension(0, 20), new java.awt.Dimension(32767, 20)); + javax.swing.JLabel interestingItemLabel = new javax.swing.JLabel(); + javax.swing.Box.Filler filler6 = new javax.swing.Box.Filler(new java.awt.Dimension(0, 2), new java.awt.Dimension(0, 2), new java.awt.Dimension(32767, 2)); + javax.swing.JPanel interestingItemPanel = interestingItemsTable; + javax.swing.Box.Filler filler3 = new javax.swing.Box.Filler(new java.awt.Dimension(0, 0), new java.awt.Dimension(0, 0), new java.awt.Dimension(0, 32767)); + + mainContentPanel.setBorder(javax.swing.BorderFactory.createEmptyBorder(10, 10, 10, 10)); + mainContentPanel.setMaximumSize(new java.awt.Dimension(32767, 452)); + mainContentPanel.setMinimumSize(new java.awt.Dimension(200, 452)); + mainContentPanel.setLayout(new javax.swing.BoxLayout(mainContentPanel, javax.swing.BoxLayout.PAGE_AXIS)); + + org.openide.awt.Mnemonics.setLocalizedText(hashsetHitsLabel, org.openide.util.NbBundle.getMessage(AnalysisPanel.class, "AnalysisPanel.hashsetHitsLabel.text")); // NOI18N + mainContentPanel.add(hashsetHitsLabel); + mainContentPanel.add(filler1); + + hashSetHitsPanel.setMaximumSize(new java.awt.Dimension(32767, 106)); + hashSetHitsPanel.setMinimumSize(new java.awt.Dimension(10, 106)); + hashSetHitsPanel.setPreferredSize(new java.awt.Dimension(32767, 106)); + mainContentPanel.add(hashSetHitsPanel); + mainContentPanel.add(filler2); + + org.openide.awt.Mnemonics.setLocalizedText(keywordHitsLabel, org.openide.util.NbBundle.getMessage(AnalysisPanel.class, "AnalysisPanel.keywordHitsLabel.text")); // NOI18N + mainContentPanel.add(keywordHitsLabel); + mainContentPanel.add(filler4); + + keywordHitsPanel.setMaximumSize(new java.awt.Dimension(32767, 106)); + keywordHitsPanel.setMinimumSize(new java.awt.Dimension(10, 106)); + keywordHitsPanel.setPreferredSize(new java.awt.Dimension(32767, 106)); + mainContentPanel.add(keywordHitsPanel); + mainContentPanel.add(filler5); + + org.openide.awt.Mnemonics.setLocalizedText(interestingItemLabel, org.openide.util.NbBundle.getMessage(AnalysisPanel.class, "AnalysisPanel.interestingItemLabel.text")); // NOI18N + mainContentPanel.add(interestingItemLabel); + mainContentPanel.add(filler6); + + interestingItemPanel.setMaximumSize(new java.awt.Dimension(32767, 106)); + interestingItemPanel.setMinimumSize(new java.awt.Dimension(10, 106)); + interestingItemPanel.setPreferredSize(new java.awt.Dimension(32767, 106)); + mainContentPanel.add(interestingItemPanel); + mainContentPanel.add(filler3); + + mainScrollPane.setViewportView(mainContentPanel); + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(mainScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 756, Short.MAX_VALUE) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(mainScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 300, Short.MAX_VALUE) + ); + }// //GEN-END:initComponents + + // Variables declaration - do not modify//GEN-BEGIN:variables + // End of variables declaration//GEN-END:variables +} From c9d7c8146e14444d4897319efa7a7e5e84c7e222 Mon Sep 17 00:00:00 2001 From: apriestman Date: Tue, 8 Sep 2020 09:00:54 -0400 Subject: [PATCH 13/24] Don't create empty groups. Fix typo. --- .../datamodel/grouping/GroupManager.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 774f6ea3e5..34cc943e34 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -365,7 +365,7 @@ public class GroupManager { } else { //group == null // It may be that this was the last unanalyzed file in the group, so test // whether the group is now fully analyzed. - return popuplateIfAnalyzed(groupKey, null); + return populateIfAnalyzed(groupKey, null); } } @@ -574,7 +574,7 @@ public class GroupManager { * 'populateIfAnalyzed' will still not return a group and therefore * this method will never mark the group as unseen. */ - group = popuplateIfAnalyzed(groupKey, null); + group = populateIfAnalyzed(groupKey, null); } else { //if there is aleady a group that was previously deemed fully analyzed, then add this newly analyzed file to it. group.addFile(fileID); @@ -680,7 +680,7 @@ public class GroupManager { } else if (groupKey.getValue().toString().equalsIgnoreCase(this.currentPathGroup.getValue().toString()) == false) { // mark the last path group as analyzed getDrawableDB().markGroupAnalyzed(currentPathGroup); - popuplateIfAnalyzed(currentPathGroup, null); + populateIfAnalyzed(currentPathGroup, null); currentPathGroup = groupKey; } @@ -698,7 +698,7 @@ public class GroupManager { try { if (currentPathGroup != null) { getDrawableDB().markGroupAnalyzed(currentPathGroup); - popuplateIfAnalyzed(currentPathGroup, null); + populateIfAnalyzed(currentPathGroup, null); currentPathGroup = null; } } catch (TskCoreException ex) { @@ -713,7 +713,7 @@ public class GroupManager { * * @returns null if Group is not ready to be viewed */ - synchronized private DrawableGroup popuplateIfAnalyzed(GroupKey groupKey, ReGroupTask task) { + synchronized private DrawableGroup populateIfAnalyzed(GroupKey groupKey, ReGroupTask task) { /* * If this method call is part of a ReGroupTask and that task is * cancelled, no-op. @@ -735,7 +735,7 @@ public class GroupManager { if (groupKey.getAttribute() != DrawableAttribute.PATH || getDrawableDB().isGroupAnalyzed(groupKey)) { Set fileIDs = getFileIDsInGroup(groupKey); - if (Objects.nonNull(fileIDs)) { + if (Objects.nonNull(fileIDs) && ! fileIDs.isEmpty()) { long examinerID = collaborativeModeProp.get() ? -1 : controller.getCaseDatabase().getCurrentExaminer().getId(); final boolean groupSeen = getDrawableDB().isGroupSeenByExaminer(groupKey, examinerID); @@ -866,7 +866,7 @@ public class GroupManager { p++; updateMessage(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString()) + valForDataSource.getValue()); updateProgress(p, valsByDataSource.size()); - popuplateIfAnalyzed(new GroupKey<>(groupBy, valForDataSource.getValue(), valForDataSource.getKey()), this); + populateIfAnalyzed(new GroupKey<>(groupBy, valForDataSource.getValue(), valForDataSource.getKey()), this); } Optional viewedGroup From 6ebb4fb3a7b66932b175e0f2e5fd8b1bde69c806 Mon Sep 17 00:00:00 2001 From: apriestman Date: Tue, 8 Sep 2020 10:28:23 -0400 Subject: [PATCH 14/24] Removed "uniquely" from domain description. --- .../src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml index 098041fb63..4cedc25d2b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml @@ -12,7 +12,7 @@ Each splitToken contains a single mapping of a raw URL substring to its regex eq SearchEngine: engine: The engines basic name - domainSubstring: The domain of the URL such that it can uniquely be identified as given engine. Should not have leading or trailing '.' + domainSubstring: The domain of the URL such that it can be identified as given engine. Should not have leading or trailing '.' splitToken: plainToken: The string in the URL that is immediately followed by the actual query. From 880f29e86836fe98ad2fec241e53afde978c4b15 Mon Sep 17 00:00:00 2001 From: apriestman Date: Tue, 8 Sep 2020 12:31:17 -0400 Subject: [PATCH 15/24] Updated xml file name --- .../autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index 27e50c19e1..0dde950a05 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -59,7 +59,7 @@ import org.xml.sax.SAXException; * artifacts, and extracting search text from them. * * - * To add search engines, edit SearchEngines.xml under RecentActivity + * To add search engines, edit SEUQAMappings.xml under RecentActivity * */ @NbBundle.Messages({ From d7f7e1f986b5749fd1d7d4c53316671d01c0214b Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Sep 2020 13:11:05 -0400 Subject: [PATCH 16/24] bug fixes --- .../autopsy/datasourcesummary/ui/AnalysisPanel.form | 9 ++++++--- .../autopsy/datasourcesummary/ui/AnalysisPanel.java | 9 ++++++--- .../datasourcesummary/ui/Bundle.properties-MERGED | 6 ++++++ .../ui/DataSourceSummaryTabbedPane.java | 6 ++++-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form index dd8ae51623..4016b539a7 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.form @@ -88,6 +88,7 @@ + @@ -95,7 +96,7 @@ - + @@ -155,6 +156,7 @@ + @@ -162,7 +164,7 @@ - + @@ -222,6 +224,7 @@ + @@ -229,7 +232,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java index f1b1483deb..8f8698316a 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java @@ -155,9 +155,10 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel { mainContentPanel.add(hashsetHitsLabel); mainContentPanel.add(filler1); + hashSetHitsPanel.setAlignmentX(0.0F); hashSetHitsPanel.setMaximumSize(new java.awt.Dimension(32767, 106)); hashSetHitsPanel.setMinimumSize(new java.awt.Dimension(10, 106)); - hashSetHitsPanel.setPreferredSize(new java.awt.Dimension(32767, 106)); + hashSetHitsPanel.setPreferredSize(new java.awt.Dimension(10, 106)); mainContentPanel.add(hashSetHitsPanel); mainContentPanel.add(filler2); @@ -165,9 +166,10 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel { mainContentPanel.add(keywordHitsLabel); mainContentPanel.add(filler4); + keywordHitsPanel.setAlignmentX(0.0F); keywordHitsPanel.setMaximumSize(new java.awt.Dimension(32767, 106)); keywordHitsPanel.setMinimumSize(new java.awt.Dimension(10, 106)); - keywordHitsPanel.setPreferredSize(new java.awt.Dimension(32767, 106)); + keywordHitsPanel.setPreferredSize(new java.awt.Dimension(10, 106)); mainContentPanel.add(keywordHitsPanel); mainContentPanel.add(filler5); @@ -175,9 +177,10 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel { mainContentPanel.add(interestingItemLabel); mainContentPanel.add(filler6); + interestingItemPanel.setAlignmentX(0.0F); interestingItemPanel.setMaximumSize(new java.awt.Dimension(32767, 106)); interestingItemPanel.setMinimumSize(new java.awt.Dimension(10, 106)); - interestingItemPanel.setPreferredSize(new java.awt.Dimension(32767, 106)); + interestingItemPanel.setPreferredSize(new java.awt.Dimension(10, 106)); mainContentPanel.add(interestingItemPanel); mainContentPanel.add(filler3); diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties-MERGED index bd71f58f9c..f3325f2e46 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/Bundle.properties-MERGED @@ -1,3 +1,5 @@ +AnalysisPanel_countColumn_title=Count +AnalysisPanel_keyColumn_title=Name CTL_DataSourceSummaryAction=Data Source Summary DataSourceSummaryCountsPanel.ArtifactCountsTableModel.count.header=Count DataSourceSummaryCountsPanel.ArtifactCountsTableModel.type.header=Result Type @@ -64,6 +66,7 @@ DataSourceSummaryNode.column.status.header=Ingest Status DataSourceSummaryNode.column.tags.header=Tags DataSourceSummaryNode.column.type.header=Type DataSourceSummaryNode.viewDataSourceAction.text=Go to Data Source +DataSourceSummaryTabbedPane_analysisTab_title=Analysis DataSourceSummaryTabbedPane_countsTab_title=Counts DataSourceSummaryTabbedPane_detailsTab_title=Container DataSourceSummaryTabbedPane_ingestHistoryTab_title=Ingest History @@ -74,6 +77,9 @@ DataSourceSummaryUserActivityPanel.recentAccountsLabel.text=Recent Accounts DataSourceSummaryUserActivityPanel.topWebSearchLabel.text=Recent Web Searches DataSourceSummaryUserActivityPanel.topDevicesAttachedLabel.text=Recent Devices Attached DataSourceSummaryUserActivityPanel.recentDomainsLabel.text=Recent Domains +AnalysisPanel.hashsetHitsLabel.text=Hashset Hits +AnalysisPanel.keywordHitsLabel.text=Keyword Hits +AnalysisPanel.interestingItemLabel.text=Interesting Item Hits DataSourceSummaryUserActivityPanel_noDataExists=No communication data exists DataSourceSummaryUserActivityPanel_tab_title=User Activity DataSourceSummaryUserActivityPanel_TopAccountTableModel_accountType_header=Account Type diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryTabbedPane.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryTabbedPane.java index 047af1cdd9..9696f4d4c4 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryTabbedPane.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryTabbedPane.java @@ -37,7 +37,8 @@ import org.sleuthkit.datamodel.DataSource; "DataSourceSummaryTabbedPane_detailsTab_title=Container", "DataSourceSummaryTabbedPane_userActivityTab_title=User Activity", "DataSourceSummaryTabbedPane_ingestHistoryTab_title=Ingest History", - "DataSourceSummaryTabbedPane_recentFileTab_title=Recent Files" + "DataSourceSummaryTabbedPane_recentFileTab_title=Recent Files", + "DataSourceSummaryTabbedPane_analysisTab_title=Analysis" }) public class DataSourceSummaryTabbedPane extends JTabbedPane { @@ -47,7 +48,8 @@ public class DataSourceSummaryTabbedPane extends JTabbedPane { private final List> tabs = new ArrayList<>(Arrays.asList( Pair.of(Bundle.DataSourceSummaryTabbedPane_countsTab_title(), new DataSourceSummaryCountsPanel()), Pair.of(Bundle.DataSourceSummaryTabbedPane_userActivityTab_title(), new DataSourceSummaryUserActivityPanel()), - Pair.of(Bundle.DataSourceSummaryTabbedPane_recentFileTab_title(), new RecentFilesPanel()) + Pair.of(Bundle.DataSourceSummaryTabbedPane_recentFileTab_title(), new RecentFilesPanel()), + Pair.of(Bundle.DataSourceSummaryTabbedPane_analysisTab_title(), new AnalysisPanel()) )); private final IngestJobInfoPanel ingestHistoryPanel = new IngestJobInfoPanel(); From 0726c1d55c2a1a5ec065067faabf1819247e9991 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Sep 2020 15:05:01 -0400 Subject: [PATCH 17/24] commenting and formatting --- .../datamodel/DataSourceAnalysisSummary.java | 103 +++++++++++++++--- .../datasourcesummary/ui/AnalysisPanel.java | 30 +++-- 2 files changed, 105 insertions(+), 28 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java index a42a46b56e..e15c1f98f7 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java @@ -19,13 +19,15 @@ package org.sleuthkit.autopsy.datasourcesummary.datamodel; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -39,40 +41,109 @@ import org.sleuthkit.datamodel.TskCoreException; * Providing data for the data source analysis tab. */ public class DataSourceAnalysisSummary { + private static final BlackboardAttribute.Type TYPE_SET_NAME = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_SET_NAME); - - private final java.util.logging.Logger logger; + + private static final Set EXCLUDED_KEYWORD_SEARCH_ITEMS = new HashSet<>(Arrays.asList( + "PHONE NUMBERS", + "IP ADDRESSES", + "EMAIL ADDRESSES", + "URLS", + "CREDIT CARD NUMBERS" + )); + private final SleuthkitCaseProvider provider; + /** + * Main constructor. + */ public DataSourceAnalysisSummary() { - this(Logger.getLogger(DataSourceAnalysisSummary.class.getName()), SleuthkitCaseProvider.DEFAULT); + this(SleuthkitCaseProvider.DEFAULT); } - - public DataSourceAnalysisSummary(java.util.logging.Logger logger, SleuthkitCaseProvider provider) { - this.logger = logger; + + /** + * Main constructor. + * + * @param provider The means of obtaining a sleuthkit case. + */ + public DataSourceAnalysisSummary(SleuthkitCaseProvider provider) { this.provider = provider; } - + + /** + * Gets counts for hashset hits. + * + * @param dataSource The datasource for which to identify hashset hits. + * + * @return The hashset set name with the number of hits in descending order. + * + * @throws + * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws TskCoreException + */ public List> getHashsetCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { return getCountsData(dataSource, TYPE_SET_NAME, ARTIFACT_TYPE.TSK_HASHSET_HIT); } + /** + * Gets counts for keyword hits. + * + * @param dataSource The datasource for which to identify keyword hits. + * + * @return The keyword set name with the number of hits in descending order. + * + * @throws + * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws TskCoreException + */ public List> getKeywordCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { - return getCountsData(dataSource, TYPE_SET_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT); + return getCountsData(dataSource, TYPE_SET_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT).stream() + // make sure we have a valid set and that that set does not belong to the set of excluded items + .filter((pair) -> pair != null && pair.getKey() != null && !EXCLUDED_KEYWORD_SEARCH_ITEMS.contains(pair.getKey().toUpperCase().trim())) + .collect(Collectors.toList()); } - + + /** + * Gets counts for interesting item hits. + * + * @param dataSource The datasource for which to identify interesting item + * hits. + * + * @return The interesting item set name with the number of hits in + * descending order. + * + * @throws + * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws TskCoreException + */ public List> getInterestingItemCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { return getCountsData(dataSource, TYPE_SET_NAME, ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT); } - - private List> getCountsData(DataSource dataSource, BlackboardAttribute.Type keyType, ARTIFACT_TYPE... artifactTypes) throws SleuthkitCaseProviderException, TskCoreException { + + /** + * Get counts for the artifact of the specified type. + * + * @param dataSource The datasource. + * @param keyType The attribute to use as the key type. + * @param artifactTypes The types of artifacts for which to query. + * + * @return A list of key value pairs where the key is the attribute type + * value and the value is the count of items found. This list is + * sorted by the count descending max to min. + * + * @throws + * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws TskCoreException + */ + private List> getCountsData(DataSource dataSource, BlackboardAttribute.Type keyType, ARTIFACT_TYPE... artifactTypes) + throws SleuthkitCaseProviderException, TskCoreException { List artifacts = new ArrayList<>(); SleuthkitCase skCase = provider.get(); - + for (ARTIFACT_TYPE type : artifactTypes) { artifacts.addAll(skCase.getBlackboard().getArtifacts(type.getTypeID(), dataSource.getId())); } - + Map countedKeys = artifacts.stream() .map((art) -> { String key = DataSourceInfoUtilities.getStringOrNull(art, keyType); @@ -80,10 +151,10 @@ public class DataSourceAnalysisSummary { }) .filter((key) -> key != null) .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); - + return countedKeys.entrySet().stream() .map((e) -> Pair.of(e.getKey(), e.getValue())) - .sorted((a,b) -> -a.getValue().compareTo(b.getValue())) + .sorted((a, b) -> -a.getValue().compareTo(b.getValue())) .collect(Collectors.toList()); } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java index 8f8698316a..16de061385 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java @@ -44,20 +44,22 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel { private static final long serialVersionUID = 1L; - private static final ColumnModel> KEY_COL = new ColumnModel<>( - Bundle.AnalysisPanel_keyColumn_title(), - (pair) -> new DefaultCellModel(pair.getKey()), - 300 + /** + * Default Column definitions for each table + */ + private static final List>> DEFAULT_COLUMNS = Arrays.asList( + new ColumnModel<>( + Bundle.AnalysisPanel_keyColumn_title(), + (pair) -> new DefaultCellModel(pair.getKey()), + 300 + ), + new ColumnModel<>( + Bundle.AnalysisPanel_countColumn_title(), + (pair) -> new DefaultCellModel(String.valueOf(pair.getValue())), + 100 + ) ); - private static final ColumnModel> COUNT_COL = new ColumnModel<>( - Bundle.AnalysisPanel_countColumn_title(), - (pair) -> new DefaultCellModel(String.valueOf(pair.getValue())), - 100 - ); - - private static final List>> DEFAULT_COLUMNS = Arrays.asList(KEY_COL, COUNT_COL); - private final JTablePanel> hashsetHitsTable = JTablePanel.getJTablePanel(DEFAULT_COLUMNS); private final JTablePanel> keywordHitsTable = JTablePanel.getJTablePanel(DEFAULT_COLUMNS); @@ -70,6 +72,10 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel { interestingItemsTable ); + /** + * All of the components necessary for data fetch swing workers to load data + * for each table. + */ private final List> dataFetchComponents; /** From 223ea6a0bee3839f8d4f18d0e507173a9c1ad334 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 8 Sep 2020 17:39:09 -0400 Subject: [PATCH 18/24] 6807 Tidy up MediaViewImagePanel --- .../contentviewers/MediaViewImagePanel.java | 113 +++++++++--------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java index 957b887c75..d45a7c057e 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2019 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -97,68 +97,70 @@ import org.sleuthkit.datamodel.TskCoreException; * Image viewer part of the Media View layered pane. Uses JavaFX to display the * image. */ -@NbBundle.Messages({"MediaViewImagePanel.externalViewerButton.text=Open in External Viewer Ctrl+E", +@NbBundle.Messages({ + "MediaViewImagePanel.externalViewerButton.text=Open in External Viewer Ctrl+E", "MediaViewImagePanel.errorLabel.text=Could not load file into Media View.", - "MediaViewImagePanel.errorLabel.OOMText=Could not load file into Media View: insufficent memory."}) + "MediaViewImagePanel.errorLabel.OOMText=Could not load file into Media View: insufficent memory." +}) @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPanel { - private static final Image EXTERNAL = new Image(MediaViewImagePanel.class.getResource("/org/sleuthkit/autopsy/images/external.png").toExternalForm()); - private final static Logger LOGGER = Logger.getLogger(MediaViewImagePanel.class.getName()); + private static final long serialVersionUID = 1L; + private static final Logger logger = Logger.getLogger(MediaViewImagePanel.class.getName()); + private static final double[] ZOOM_STEPS = { + 0.0625, 0.125, 0.25, 0.375, 0.5, 0.75, + 1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10}; + private static final double MIN_ZOOM_RATIO = 0.0625; // 6.25% + private static final double MAX_ZOOM_RATIO = 10.0; // 1000% + private static final Image externalImage = new Image(MediaViewImagePanel.class.getResource("/org/sleuthkit/autopsy/images/external.png").toExternalForm()); + private static final SortedSet supportedMimes = ImageUtils.getSupportedImageMimeTypes(); + private static final List supportedExtensions = ImageUtils.getSupportedImageExtensions().stream() + .map("."::concat) //NOI18N + .collect(Collectors.toList()); private final boolean fxInited; - - private JFXPanel fxPanel; - private AbstractFile file; - private Group masterGroup; - private ImageTagsGroup tagsGroup; - private ImageTagCreator imageTagCreator; - private ImageView fxImageView; - private ScrollPane scrollPane; + + /* + * JFX + */ private final ProgressBar progressBar = new ProgressBar(); private final MaskerPane maskerPane = new MaskerPane(); - + + /* + * Swing + */ private final JPopupMenu imageTaggingOptions = new JPopupMenu(); private final JMenuItem createTagMenuItem; private final JMenuItem deleteTagMenuItem; private final JMenuItem hideTagsMenuItem; private final JMenuItem exportTagsMenuItem; - private final JFileChooser exportChooser; - private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); + /* + * JFX + */ + private Group masterGroup; + private ImageTagsGroup tagsGroup; + private ImageTagCreator imageTagCreator; + private ImageView fxImageView; + private ScrollPane scrollPane; + private Task readImageTask; + + /* + * Swing + */ + private JFXPanel fxPanel; + private double zoomRatio; private double rotation; // Can be 0, 90, 180, and 270. - - private boolean autoResize = true; // Auto resize when the user changes the size - // of the content viewer unless the user has used the zoom buttons. - private static final double[] ZOOM_STEPS = { - 0.0625, 0.125, 0.25, 0.375, 0.5, 0.75, - 1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10}; - - private static final double MIN_ZOOM_RATIO = 0.0625; // 6.25% - private static final double MAX_ZOOM_RATIO = 10.0; // 1000% + private boolean autoResize = true; // Auto resize when the user changes the size of the content viewer unless the user has used the zoom buttons. + private AbstractFile file; static { ImageIO.scanForPlugins(); } - /** - * mime types we should be able to display. if the mimetype is unknown we - * will fall back on extension and jpg/png header - */ - static private final SortedSet supportedMimes = ImageUtils.getSupportedImageMimeTypes(); - - /** - * extensions we should be able to display - */ - static private final List supportedExtensions = ImageUtils.getSupportedImageExtensions().stream() - .map("."::concat) //NOI18N - .collect(Collectors.toList()); - - private Task readImageTask; - /** * Creates new form MediaViewImagePanel */ @@ -168,7 +170,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan "MediaViewImagePanel.hideTagOption=Hide", "MediaViewImagePanel.exportTagOption=Export" }) - public MediaViewImagePanel() { + MediaViewImagePanel() { initComponents(); fxInited = org.sleuthkit.autopsy.core.Installer.isJavaFxInited(); @@ -354,14 +356,13 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan } private void showErrorNode(String errorMessage, AbstractFile file) { - final Button externalViewerButton = new Button(Bundle.MediaViewImagePanel_externalViewerButton_text(), new ImageView(EXTERNAL)); - externalViewerButton.setOnAction(actionEvent - -> //fx ActionEvent - /* - * TODO: why is the name passed into the action constructor? it - * means we duplicate this string all over the place -jm - */ new ExternalViewerAction(Bundle.MediaViewImagePanel_externalViewerButton_text(), new FileNode(file)) - .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) //Swing ActionEvent + final Button externalViewerButton = new Button(Bundle.MediaViewImagePanel_externalViewerButton_text(), new ImageView(externalImage)); + /* + * TODO: why is the name passed into the action constructor? it means we + * duplicate this string all over the place -jm + */ + externalViewerButton.setOnAction(actionEvent -> new ExternalViewerAction(Bundle.MediaViewImagePanel_externalViewerButton_text(), new FileNode(file)) + .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) //Swing ActionEvent ); final VBox errorNode = new VBox(10, new Label(errorMessage), externalViewerButton); @@ -420,7 +421,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan "state", null, State.NONEMPTY)); } } catch (TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not retrieve image tags for file in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not retrieve image tags for file in case db", ex); //NON-NLS } scrollPane.setContent(masterGroup); } else { @@ -693,14 +694,14 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan private void rotateLeftButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_rotateLeftButtonActionPerformed autoResize = false; - + rotation = (rotation + 270) % 360; updateView(); }//GEN-LAST:event_rotateLeftButtonActionPerformed private void rotateRightButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_rotateRightButtonActionPerformed autoResize = false; - + rotation = (rotation + 90) % 360; updateView(); }//GEN-LAST:event_rotateRightButtonActionPerformed @@ -760,7 +761,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan Case.getCurrentCase().getServices().getTagsManager().deleteContentTag(contentViewerTag.getContentTag()); tagsGroup.getChildren().remove(tagInFocus); } catch (TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not delete image tag in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not delete image tag in case db", ex); //NON-NLS } scrollPane.setCursor(Cursor.DEFAULT); @@ -793,7 +794,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan ImageTag imageTag = buildImageTag(contentViewerTag); tagsGroup.getChildren().add(imageTag); } catch (TskCoreException | SerializationException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not save new image tag in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not save new image tag in case db", ex); //NON-NLS } scrollPane.setCursor(Cursor.DEFAULT); @@ -832,7 +833,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan ImageTagRegion newRegion = (ImageTagRegion) edit.getNewValue(); ContentViewerTagManager.updateTag(contentViewerTag, newRegion); } catch (SerializationException | TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not save edit for image tag in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not save edit for image tag in case db", ex); //NON-NLS } scrollPane.setCursor(Cursor.DEFAULT); }); @@ -916,7 +917,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan JOptionPane.showMessageDialog(null, Bundle.MediaViewImagePanel_successfulExport()); } catch (Exception ex) { //Runtime exceptions may spill out of ImageTagsUtil from JavaFX. //This ensures we (devs and users) have something when it doesn't work. - LOGGER.log(Level.WARNING, "Unable to export tagged image to disk", ex); //NON-NLS + logger.log(Level.WARNING, "Unable to export tagged image to disk", ex); //NON-NLS JOptionPane.showMessageDialog(null, Bundle.MediaViewImagePanel_unsuccessfulExport()); } return null; From b2f77fd554588ae2bb8c5281959130c734aa33e4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 8 Sep 2020 17:52:32 -0400 Subject: [PATCH 19/24] 6807 Tidy up MediaViewImagePanel --- .../autopsy/contentviewers/MediaViewImagePanel.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java index d45a7c057e..ba5f1ef8e9 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java @@ -358,11 +358,11 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan private void showErrorNode(String errorMessage, AbstractFile file) { final Button externalViewerButton = new Button(Bundle.MediaViewImagePanel_externalViewerButton_text(), new ImageView(externalImage)); /* - * TODO: why is the name passed into the action constructor? it means we - * duplicate this string all over the place -jm + * Tie a Swing action (ExternalViewerAction) to a JFX button action. */ - externalViewerButton.setOnAction(actionEvent -> new ExternalViewerAction(Bundle.MediaViewImagePanel_externalViewerButton_text(), new FileNode(file)) - .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) //Swing ActionEvent + externalViewerButton.setOnAction(actionEvent -> + new ExternalViewerAction(Bundle.MediaViewImagePanel_externalViewerButton_text(), new FileNode(file)) + .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) ); final VBox errorNode = new VBox(10, new Label(errorMessage), externalViewerButton); From 91613720c360b83e07ae939b839b06584c4911c1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Sep 2020 18:48:04 -0400 Subject: [PATCH 20/24] commenting --- .../datasourcesummary/datamodel/DataSourceAnalysisSummary.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java index e15c1f98f7..9c0c791673 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java @@ -140,10 +140,12 @@ public class DataSourceAnalysisSummary { List artifacts = new ArrayList<>(); SleuthkitCase skCase = provider.get(); + // get all artifacts in one list for each artifact type for (ARTIFACT_TYPE type : artifactTypes) { artifacts.addAll(skCase.getBlackboard().getArtifacts(type.getTypeID(), dataSource.getId())); } + // group those based on the value of the attribute type that should serve as a key Map countedKeys = artifacts.stream() .map((art) -> { String key = DataSourceInfoUtilities.getStringOrNull(art, keyType); @@ -152,6 +154,7 @@ public class DataSourceAnalysisSummary { .filter((key) -> key != null) .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + // sort from max to min counts return countedKeys.entrySet().stream() .map((e) -> Pair.of(e.getKey(), e.getValue())) .sorted((a, b) -> -a.getValue().compareTo(b.getValue())) From a5849a77b00c8fd86bfbe021bfaf9002813c39f4 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Sep 2020 19:02:32 -0400 Subject: [PATCH 21/24] updated tooltip to show full folder path for folder column --- .../ui/DataSourceSummaryUserActivityPanel.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java index 309b70008e..99bae253a5 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java @@ -66,7 +66,7 @@ import org.sleuthkit.datamodel.DataSource; "DataSourceSummaryUserActivityPanel_TopAccountTableModel_accountType_header=Account Type", "DataSourceSummaryUserActivityPanel_TopAccountTableModel_lastAccess_header=Last Accessed",}) public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPanel { - + private static final long serialVersionUID = 1L; private static final DateFormat DATETIME_FORMAT = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.getDefault()); private static final int TOP_PROGS_COUNT = 10; @@ -103,7 +103,8 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan return new DefaultCellModel( getShortFolderName( prog.getProgramPath(), - prog.getProgramName())); + prog.getProgramName())) + .setTooltip(prog.getProgramPath()); }, 150), // run count column @@ -206,7 +207,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan 150 ) )); - + private final List> tables = Arrays.asList( topProgramsTable, recentDomainsTable, @@ -214,7 +215,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan topDevicesAttachedTable, topAccountsTable ); - + private final List> dataFetchComponents; private final DataSourceTopProgramsSummary topProgramsData; @@ -235,7 +236,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan public DataSourceSummaryUserActivityPanel( DataSourceTopProgramsSummary topProgramsData, DataSourceUserActivitySummary userActivityData) { - + this.topProgramsData = topProgramsData; // set up data acquisition methods @@ -266,7 +267,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan (result) -> topAccountsTable.showDataFetchResult(result, JTablePanel.getDefaultErrorMessage(), Bundle.DataSourceSummaryUserActivityPanel_noDataExists())) ); - + initComponents(); } @@ -281,7 +282,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan private String getShortFolderName(String path, String appName) { return this.topProgramsData.getShortFolderName(path, appName); } - + @Override protected void onNewDataSource(DataSource dataSource) { // if no data source is present or the case is not open, @@ -289,7 +290,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan if (dataSource == null || !Case.isCaseOpen()) { this.dataFetchComponents.forEach((item) -> item.getResultHandler() .accept(DataFetchResult.getSuccessResult(null))); - + } else { // set tables to display loading screen this.tables.forEach((table) -> table.showDefaultLoadingMessage()); From f346a1de3a63eaa52b6b526351b3c836179adf99 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 9 Sep 2020 12:01:13 -0400 Subject: [PATCH 22/24] 6807 Tidy up MediaViewImagePanel --- .../contentviewers/MediaViewImagePanel.java | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java index ba5f1ef8e9..55163535cb 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java @@ -117,17 +117,21 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan private static final List supportedExtensions = ImageUtils.getSupportedImageExtensions().stream() .map("."::concat) //NOI18N .collect(Collectors.toList()); - - private final boolean fxInited; /* - * JFX + * JFX components */ private final ProgressBar progressBar = new ProgressBar(); private final MaskerPane maskerPane = new MaskerPane(); + private Group masterGroup; + private ImageTagsGroup tagsGroup; + private ImageTagCreator imageTagCreator; + private ImageView fxImageView; + private ScrollPane scrollPane; + private Task readImageTask; /* - * Swing + * Swing components */ private final JPopupMenu imageTaggingOptions = new JPopupMenu(); private final JMenuItem createTagMenuItem; @@ -136,22 +140,12 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan private final JMenuItem exportTagsMenuItem; private final JFileChooser exportChooser; private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); - - /* - * JFX - */ - private Group masterGroup; - private ImageTagsGroup tagsGroup; - private ImageTagCreator imageTagCreator; - private ImageView fxImageView; - private ScrollPane scrollPane; - private Task readImageTask; - - /* - * Swing - */ private JFXPanel fxPanel; + /* + * State + */ + private final boolean fxInited; private double zoomRatio; private double rotation; // Can be 0, 90, 180, and 270. private boolean autoResize = true; // Auto resize when the user changes the size of the content viewer unless the user has used the zoom buttons. From 8a831b33454305500d4e1011671a7904a7e50c75 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 9 Sep 2020 14:13:29 -0400 Subject: [PATCH 23/24] exception to just class name --- .../datamodel/DataSourceAnalysisSummary.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java index 9c0c791673..86aa21c7a1 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceAnalysisSummary.java @@ -77,8 +77,7 @@ public class DataSourceAnalysisSummary { * * @return The hashset set name with the number of hits in descending order. * - * @throws - * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws SleuthkitCaseProviderException * @throws TskCoreException */ public List> getHashsetCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { @@ -92,8 +91,7 @@ public class DataSourceAnalysisSummary { * * @return The keyword set name with the number of hits in descending order. * - * @throws - * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws SleuthkitCaseProviderException * @throws TskCoreException */ public List> getKeywordCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { @@ -112,8 +110,7 @@ public class DataSourceAnalysisSummary { * @return The interesting item set name with the number of hits in * descending order. * - * @throws - * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws SleuthkitCaseProviderException * @throws TskCoreException */ public List> getInterestingItemCounts(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { @@ -131,8 +128,7 @@ public class DataSourceAnalysisSummary { * value and the value is the count of items found. This list is * sorted by the count descending max to min. * - * @throws - * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + * @throws SleuthkitCaseProviderException * @throws TskCoreException */ private List> getCountsData(DataSource dataSource, BlackboardAttribute.Type keyType, ARTIFACT_TYPE... artifactTypes) From 65c0c6fbcae6eca8e2b3fd9aa53a80da0f35c358 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 9 Sep 2020 14:32:13 -0400 Subject: [PATCH 24/24] formatting fix --- .../ui/DataSourceSummaryUserActivityPanel.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java index 99bae253a5..b4011fe4d6 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/DataSourceSummaryUserActivityPanel.java @@ -66,7 +66,7 @@ import org.sleuthkit.datamodel.DataSource; "DataSourceSummaryUserActivityPanel_TopAccountTableModel_accountType_header=Account Type", "DataSourceSummaryUserActivityPanel_TopAccountTableModel_lastAccess_header=Last Accessed",}) public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPanel { - + private static final long serialVersionUID = 1L; private static final DateFormat DATETIME_FORMAT = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.getDefault()); private static final int TOP_PROGS_COUNT = 10; @@ -207,7 +207,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan 150 ) )); - + private final List> tables = Arrays.asList( topProgramsTable, recentDomainsTable, @@ -215,7 +215,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan topDevicesAttachedTable, topAccountsTable ); - + private final List> dataFetchComponents; private final DataSourceTopProgramsSummary topProgramsData; @@ -236,7 +236,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan public DataSourceSummaryUserActivityPanel( DataSourceTopProgramsSummary topProgramsData, DataSourceUserActivitySummary userActivityData) { - + this.topProgramsData = topProgramsData; // set up data acquisition methods @@ -267,7 +267,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan (result) -> topAccountsTable.showDataFetchResult(result, JTablePanel.getDefaultErrorMessage(), Bundle.DataSourceSummaryUserActivityPanel_noDataExists())) ); - + initComponents(); } @@ -282,7 +282,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan private String getShortFolderName(String path, String appName) { return this.topProgramsData.getShortFolderName(path, appName); } - + @Override protected void onNewDataSource(DataSource dataSource) { // if no data source is present or the case is not open, @@ -290,7 +290,7 @@ public class DataSourceSummaryUserActivityPanel extends BaseDataSourceSummaryPan if (dataSource == null || !Case.isCaseOpen()) { this.dataFetchComponents.forEach((item) -> item.getResultHandler() .accept(DataFetchResult.getSuccessResult(null))); - + } else { // set tables to display loading screen this.tables.forEach((table) -> table.showDefaultLoadingMessage());