From e73581143a1824263fc18824f73130c8963a989b Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Tue, 12 Aug 2014 17:52:51 -0700 Subject: [PATCH 1/5] minor comments, added stats structures from scheduler and jobs, updated status panel to display job status --- .../ingest/DataSourceIngestPipeline.java | 6 +- .../autopsy/ingest/FileIngestPipeline.java | 4 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 146 ++++++++++++++++-- .../autopsy/ingest/IngestManager.java | 25 ++- .../ingest/IngestProgressSnapshotPanel.form | 27 +++- .../ingest/IngestProgressSnapshotPanel.java | 109 ++++++++++++- .../autopsy/ingest/IngestScheduler.java | 119 +++++++++++++- 7 files changed, 403 insertions(+), 33 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 2752646947..aec0d9042f 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -27,14 +27,14 @@ import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * A data source ingest pipeline composed of a sequence of data source ingest - * modules constructed from ingest module templates. + * This class manages a sequence of data source ingest modules. It starts them, + * shuts them down, and runs them in sequential order. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobContext context; - private List modules = new ArrayList<>(); + private final List modules = new ArrayList<>(); DataSourceIngestPipeline(IngestJobContext context, List moduleTemplates) { this.context = context; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 573cf96a06..02dd61b7c7 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -23,8 +23,8 @@ import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * A file ingest pipeline composed of a sequence of file ingest modules - * constructed from ingest module templates. + * This class manages a sequence of file ingest modules. It starts them, + * shuts them down, and runs a file through them. */ final class FileIngestPipeline { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index f885b5c171..5b4d9f388d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.Level; @@ -45,15 +46,20 @@ final class IngestJob { private final LinkedBlockingQueue fileIngestPipelines = new LinkedBlockingQueue<>(); private long estimatedFilesToProcess = 0L; // Guarded by this private long processedFiles = 0L; // Guarded by this + // Contains names of files that are being run. Used to keep progress bar up to date. + private final List filesInProgress = new ArrayList<>(); // Guarded by this. private DataSourceIngestPipeline dataSourceIngestPipeline; private ProgressHandle dataSourceIngestProgress; private ProgressHandle fileIngestProgress; private volatile boolean cancelled = false; + private long startTime; + IngestJob(long id, Content dataSource, boolean processUnallocatedSpace) { this.id = id; this.dataSource = dataSource; this.processUnallocatedSpace = processUnallocatedSpace; + startTime = new Date().getTime(); } long getId() { @@ -139,20 +145,14 @@ final class IngestJob { dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { - if (dataSourceIngestProgress != null) { - dataSourceIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), - "IngestJob.progress.cancelling", - displayName)); - } - IngestScheduler.getInstance().cancelIngestJob(IngestJob.this); + handleCancelFromProgressBar(); return true; } }); dataSourceIngestProgress.start(); dataSourceIngestProgress.switchToIndeterminate(); } - + private void startFileIngestProgressBar() { final String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", @@ -160,12 +160,7 @@ final class IngestJob { fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { - if (fileIngestProgress != null) { - fileIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", - displayName)); - } - IngestScheduler.getInstance().cancelIngestJob(IngestJob.this); + handleCancelFromProgressBar(); return true; } }); @@ -173,6 +168,34 @@ final class IngestJob { fileIngestProgress.start(); fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess); } + + /** + * Common steps to cancel a job, regardless of what progress bar was + * used to cancel the job. + */ + private void handleCancelFromProgressBar() { + cancel(); + + if (fileIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + dataSource.getName()); + fileIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", + displayName)); + } + + if (dataSourceIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + dataSource.getName()); + dataSourceIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), + "IngestJob.progress.cancelling", + displayName)); + } + IngestScheduler.getInstance().cancelIngestJob(IngestJob.this); + } /** * Check to see if the job has a data source ingest pipeline. @@ -192,6 +215,11 @@ final class IngestJob { return (fileIngestPipelines.peek().isEmpty() == false); } + /** + * Executes the data source ingest module pipeline for the data source in the task + * @param task + * @throws InterruptedException + */ void process(DataSourceIngestTask task) throws InterruptedException { try { if (!isCancelled() && !dataSourceIngestPipeline.isEmpty()) { @@ -201,6 +229,8 @@ final class IngestJob { logIngestModuleErrors(errors); } } + + // all data source ingest jobs are done for this task, so shut down progress bar if (null != dataSourceIngestProgress) { dataSourceIngestProgress.finish(); // This is safe because this method will be called at most once per @@ -215,6 +245,12 @@ final class IngestJob { } } + + /** + * Executes the file ingest module pipeline for the file in the task + * @param task + * @throws InterruptedException + */ void process(FileIngestTask task) throws InterruptedException { try { if (!isCancelled()) { @@ -228,12 +264,25 @@ final class IngestJob { } else { fileIngestProgress.progress(file.getName(), (int) estimatedFilesToProcess); } + filesInProgress.add(file.getName()); } + List errors = new ArrayList<>(); errors.addAll(pipeline.process(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } + + // update the progress bar in case this file was being displayed + synchronized (this) { + filesInProgress.remove(file.getName()); + if (filesInProgress.size() > 0) { + fileIngestProgress.progress(filesInProgress.get(0)); + } + else { + fileIngestProgress.progress(""); + } + } } fileIngestPipelines.put(pipeline); } @@ -243,6 +292,10 @@ final class IngestJob { } } + /** + * Shutdown pipelines and progress bars + * + */ void finish() { List errors = new ArrayList<>(); while (!fileIngestPipelines.isEmpty()) { @@ -252,9 +305,13 @@ final class IngestJob { if (!errors.isEmpty()) { logIngestModuleErrors(errors); } + + // NOTE: Nothing to do for datasource ingest modules -- no shutdown method + if (dataSourceIngestProgress != null) { dataSourceIngestProgress.finish(); - } + } + if (fileIngestProgress != null) { fileIngestProgress.finish(); } @@ -273,4 +330,63 @@ final class IngestJob { void cancel() { cancelled = true; } + + /** + * Stores basic info for a given ingest job. + */ + class IngestJobStats { + private final long startTime; + private final long processedFiles; + private final long estimatedFilesToProcess; + private final long snapShotTime; + + IngestJobStats () { + synchronized (IngestJob.this) { + this.startTime = IngestJob.this.startTime; + this.processedFiles = IngestJob.this.processedFiles; + this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; + snapShotTime = new Date().getTime(); + } + } + + /** + * Get files per second throughput since job started + * @return + */ + double getSpeed() { + return (double)processedFiles / ((snapShotTime - startTime)/1000); + } + + long getStartTime() { + return startTime; + } + + /** + * Get time these stats were collected + * @return + */ + long getSnapshotTime() { + return snapShotTime; + } + + /** + * Number of files processed by job so far + * @return + */ + long getFilesProcessed() { + return processedFiles; + } + + long getFilesEstimated() { + return estimatedFilesToProcess; + } + } + + /** + * Get some basic performance stats on this job + * @return + */ + IngestJobStats getStats() { + return new IngestJobStats(); + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3c1e23cb83..e9a0fabe2e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -197,12 +197,12 @@ public class IngestManager { return IngestScheduler.getInstance().ingestJobsAreRunning(); } - void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { - ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), ingestModuleDisplayName, task.getDataSource())); + void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { + ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource())); } void setIngestTaskProgress(FileIngestTask task, String ingestModuleDisplayName) { - ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), ingestModuleDisplayName, task.getDataSource(), task.getFile())); + ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile())); } void setIngestTaskProgressCompleted(DataSourceIngestTask task) { @@ -615,31 +615,42 @@ public class IngestManager { private final String activity; private final String dataSourceName; private final String fileName; + private final long jobId; + // nothing is running on the thread IngestThreadActivitySnapshot(long threadId) { this.threadId = threadId; startTime = new Date(); this.activity = NbBundle.getMessage(this.getClass(), "IngestManager.IngestThreadActivitySnapshot.idleThread"); this.dataSourceName = ""; this.fileName = ""; + this.jobId = 0; } - IngestThreadActivitySnapshot(long threadId, String activity, Content dataSource) { + // data souce thread + IngestThreadActivitySnapshot(long threadId, long jobId, String activity, Content dataSource) { this.threadId = threadId; + this.jobId = jobId; startTime = new Date(); this.activity = activity; this.dataSourceName = dataSource.getName(); - this.fileName = ""; + this.fileName = ""; } - - IngestThreadActivitySnapshot(long threadId, String activity, Content dataSource, AbstractFile file) { + + // file ingest thread + IngestThreadActivitySnapshot(long threadId, long jobId, String activity, Content dataSource, AbstractFile file) { this.threadId = threadId; + this.jobId = jobId; startTime = new Date(); this.activity = activity; this.dataSourceName = dataSource.getName(); this.fileName = file.getName(); } + long getJobId() { + return jobId; + } + long getThreadId() { return threadId; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form index bef05de400..cf8122d2a3 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form @@ -27,6 +27,7 @@ + @@ -36,7 +37,9 @@ - + + + @@ -71,6 +74,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 601e95f887..e224aa8656 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -18,28 +18,36 @@ */ package org.sleuthkit.autopsy.ingest; +import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; +import java.util.Enumeration; import java.util.List; import javax.swing.JDialog; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableColumn; import org.apache.commons.lang3.time.DurationFormatUtils; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.ingest.IngestJob.IngestJobStats; +import org.sleuthkit.autopsy.ingest.IngestScheduler.IngestJobSchedulerStats; public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final JDialog parent; private final IngestThreadActivitySnapshotsTableModel threadActivityTableModel; + private final IngestJobTableModel jobTableModel; IngestProgressSnapshotPanel(JDialog parent) { this.parent = parent; threadActivityTableModel = new IngestThreadActivitySnapshotsTableModel(); + jobTableModel = new IngestJobTableModel(); initComponents(); customizeComponents(); } private void customizeComponents() { threadActivitySnapshotsTable.setModel(threadActivityTableModel); + jobsTable.setModel(jobTableModel); int width = snapshotsScrollPane.getPreferredSize().width; for (int i = 0; i < threadActivitySnapshotsTable.getColumnCount(); ++i) { @@ -76,7 +84,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final String[] columnNames = {NbBundle.getMessage(this.getClass(), "IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.threadID"), - NbBundle.getMessage(this.getClass(), + NbBundle.getMessage(this.getClass(), "IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.activity"), NbBundle.getMessage(this.getClass(), "IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.dataSource"), @@ -85,7 +93,8 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { NbBundle.getMessage(this.getClass(), "IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.startTime"), NbBundle.getMessage(this.getClass(), - "IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.elapsedTime")}; + "IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.elapsedTime"), + "Job ID"}; private List snapshots; private IngestThreadActivitySnapshotsTableModel() { @@ -137,6 +146,80 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { long elapsedTime = now.getTime() - snapshot.getStartTime().getTime(); cellValue = DurationFormatUtils.formatDurationHMS(elapsedTime); break; + case 6: + cellValue = snapshot.getJobId(); + break; + default: + cellValue = null; + break; + } + return cellValue; + } + } + + private class IngestJobTableModel extends AbstractTableModel { + + private final String[] columnNames = {"Job ID", + "Data Source", "Start", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; + private List schedStats; + + private IngestJobTableModel() { + refresh(); + } + + private void refresh() { + schedStats = IngestScheduler.getInstance().getJobStats(); + fireTableDataChanged(); + } + + @Override + public int getRowCount() { + return schedStats.size(); + } + + @Override + public int getColumnCount() { + return columnNames.length; + } + + @Override + public String getColumnName(int col) { + return columnNames[col]; + } + + @Override + public Object getValueAt(int rowIndex, int columnIndex) { + IngestJobSchedulerStats schedStat = schedStats.get(rowIndex); + Object cellValue; + switch (columnIndex) { + case 0: + cellValue = schedStat.getJobId(); + break; + case 1: + cellValue = schedStat.getDataSource(); + break; + case 2: + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + cellValue = dateFormat.format(new Date(schedStat.ingestJobStats.getStartTime())); + break; + case 3: + cellValue = schedStat.ingestJobStats.getSpeed(); + break; + case 4: + cellValue = schedStat.getRunningListSize(); + break; + case 5: + cellValue = schedStat.getFileQueueSize(); + break; + case 6: + cellValue = schedStat.getDirQueueSize(); + break; + case 7: + cellValue = schedStat.getRootQueueSize(); + break; + case 8: + cellValue = schedStat.getDsQueueSize(); + break; default: cellValue = null; break; @@ -156,6 +239,8 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { snapshotsScrollPane = new javax.swing.JScrollPane(); threadActivitySnapshotsTable = new javax.swing.JTable(); + jobsScrollPane = new javax.swing.JScrollPane(); + jobsTable = new javax.swing.JTable(); refreshButton = new javax.swing.JButton(); closeButton = new javax.swing.JButton(); fileProcessedPerSecondLabel = new javax.swing.JLabel(); @@ -170,6 +255,16 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { )); snapshotsScrollPane.setViewportView(threadActivitySnapshotsTable); + jobsTable.setModel(new javax.swing.table.DefaultTableModel( + new Object [][] { + + }, + new String [] { + + } + )); + jobsScrollPane.setViewportView(jobsTable); + org.openide.awt.Mnemonics.setLocalizedText(refreshButton, org.openide.util.NbBundle.getMessage(IngestProgressSnapshotPanel.class, "IngestProgressSnapshotPanel.refreshButton.text")); // NOI18N refreshButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -199,7 +294,8 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addComponent(refreshButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(closeButton))) + .addComponent(closeButton)) + .addComponent(jobsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 881, Short.MAX_VALUE)) .addContainerGap()) ); @@ -209,7 +305,9 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addContainerGap() - .addComponent(snapshotsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 316, Short.MAX_VALUE) + .addComponent(snapshotsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 155, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(jobsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 155, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(refreshButton) @@ -228,6 +326,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private void refreshButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_refreshButtonActionPerformed threadActivityTableModel.refresh(); + jobTableModel.refresh(); IngestManager.ProcessedFilesSnapshot snapshot = IngestManager.getInstance().getProcessedFilesSnapshot(); Date now = new Date(); long elapsedTime = now.getTime() - snapshot.getStartTime().getTime(); @@ -238,6 +337,8 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton closeButton; private javax.swing.JLabel fileProcessedPerSecondLabel; + private javax.swing.JScrollPane jobsScrollPane; + private javax.swing.JTable jobsTable; private javax.swing.JButton refreshButton; private javax.swing.JScrollPane snapshotsScrollPane; private javax.swing.JTable threadActivitySnapshotsTable; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index b5338cd773..13a599c6a1 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -32,13 +33,14 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.IngestJob.IngestJobStats; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.autopsy.coreutils.Logger; /** * Creates ingest jobs and their constituent ingest tasks, enabled the tasks for @@ -364,6 +366,12 @@ final class IngestScheduler { return !ingestJobsById.isEmpty(); } + /** + * Clears the queues for the job. If job is complete, finish it up. Otherwise, + * this will exit and the last worker thread will finish / clean up. + * + * @param job + */ synchronized void cancelIngestJob(IngestJob job) { long jobId = job.getId(); removeAllPendingTasksForJob(pendingRootDirectoryTasks, jobId); @@ -376,6 +384,25 @@ final class IngestScheduler { } } + /** + * Return the number of tasks in the queue for the given job ID + * @param + * @param queue + * @param jobId + * @return + */ + int countJobsInCollection(Collection queue, long jobId) { + Iterator iterator = queue.iterator(); + int count = 0; + while (iterator.hasNext()) { + IngestTask task = (IngestTask) iterator.next(); + if (task.getIngestJob().getId() == jobId) { + count++; + } + } + return count; + } + synchronized private void removeAllPendingTasksForJob(Collection taskQueue, long jobId) { Iterator iterator = taskQueue.iterator(); while (iterator.hasNext()) { @@ -433,6 +460,10 @@ final class IngestScheduler { return true; } + /** + * Called after all work is completed to free resources. + * @param job + */ private void finishIngestJob(IngestJob job) { job.finish(); long jobId = job.getId(); @@ -559,4 +590,90 @@ final class IngestScheduler { return task; } } + + /** + * Stores basic stats for a given job + */ + class IngestJobSchedulerStats { + IngestJobStats ingestJobStats; + private long jobId; + private String dataSource; + private long rootQueueSize; + private long dirQueueSize; + private long fileQueueSize; + private long dsQueueSize; + private long runningListSize; + + IngestJobSchedulerStats(IngestJob job) { + ingestJobStats = job.getStats(); + jobId = job.getId(); + dataSource = job.getDataSource().getName(); + rootQueueSize = countJobsInCollection(pendingRootDirectoryTasks, jobId); + dirQueueSize = countJobsInCollection(pendingDirectoryTasks, jobId); + fileQueueSize = countJobsInCollection(pendingFileTasks, jobId); + dsQueueSize = countJobsInCollection(pendingDataSourceTasks, jobId); + runningListSize = countJobsInCollection(tasksInProgressAndPending, jobId) - fileQueueSize - dsQueueSize; + } + + /** + * @return the jobId + */ + protected long getJobId() { + return jobId; + } + + /** + * @return the dataSource + */ + protected String getDataSource() { + return dataSource; + } + + /** + * @return the rootQueueSize + */ + protected long getRootQueueSize() { + return rootQueueSize; + } + + /** + * @return the dirQueueSize + */ + protected long getDirQueueSize() { + return dirQueueSize; + } + + /** + * @return the fileQueueSize + */ + protected long getFileQueueSize() { + return fileQueueSize; + } + + /** + * @return the dsQueueSize + */ + protected long getDsQueueSize() { + return dsQueueSize; + } + + /** + * @return the runningListSize + */ + protected long getRunningListSize() { + return runningListSize; + } + } + + /** + * Get basic performance / stats on all running jobs + * @return + */ + synchronized List getJobStats() { + List stats = new ArrayList<>(); + for (IngestJob job : Collections.list(ingestJobsById.elements())) { + stats.add(new IngestJobSchedulerStats(job)); + } + return stats; + } } From e22593dcfc0d9acbd5923875ecf961c73c100287 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 13 Aug 2014 05:06:35 -0700 Subject: [PATCH 2/5] Added diagnostics panel to get throughput numbers for debugging performance issues --- .../autopsy/diagnostics/Bundle.properties | 11 + .../autopsy/diagnostics/PerformancePanel.form | 184 +++++++ .../autopsy/diagnostics/PerformancePanel.java | 500 ++++++++++++++++++ .../diagnostics/PerformancePanelAction.java | 46 ++ .../ingest/IngestProgressSnapshotPanel.java | 3 - 5 files changed, 741 insertions(+), 3 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties create mode 100755 Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.form create mode 100755 Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java create mode 100755 Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties new file mode 100755 index 0000000000..eae8c24793 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties @@ -0,0 +1,11 @@ +PerformancePanel.jLabel1.text=Image Reading: +PerformancePanel.jLabel2.text=Database Reading: +PerformancePanel.jLabel3.text=CPU: +PerformancePanel.imgReadLabel.text=\ +PerformancePanel.dbReadLabel.text=\ +PerformancePanel.cpuTimeLabel.text=\ +PerformancePanel.startButton.text=Start +PerformancePanel.statusLabel.text=\ +PerformancePanel.jLabel4.text=File Reading: +PerformancePanel.fileReadLabel.text=\ +PerformancePanel.jLabel5.text=This panel performs a series of tests to help identify bottlenecks in the system. diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.form b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.form new file mode 100755 index 0000000000..7d0049cd82 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.form @@ -0,0 +1,184 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java new file mode 100755 index 0000000000..55ea404b2e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java @@ -0,0 +1,500 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.sleuthkit.autopsy.diagnostics; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import javax.swing.JFrame; +import javax.swing.SwingUtilities; +import javax.swing.SwingWorker; +import org.openide.util.Exceptions; +import org.openide.windows.WindowManager; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Image; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * + * @author brianc + */ +public class PerformancePanel extends javax.swing.JDialog { + + /** + * Creates new form PerformancePanel + */ + public PerformancePanel() { + super((JFrame) WindowManager.getDefault().getMainWindow(), "Performance Diagnostics", true); + initComponents(); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + jLabel1 = new javax.swing.JLabel(); + imgReadLabel = new javax.swing.JLabel(); + jLabel2 = new javax.swing.JLabel(); + dbReadLabel = new javax.swing.JLabel(); + jLabel4 = new javax.swing.JLabel(); + fileReadLabel = new javax.swing.JLabel(); + jLabel3 = new javax.swing.JLabel(); + cpuTimeLabel = new javax.swing.JLabel(); + startButton = new javax.swing.JButton(); + statusLabel = new javax.swing.JLabel(); + jLabel5 = new javax.swing.JLabel(); + + setDefaultCloseOperation(javax.swing.WindowConstants.DISPOSE_ON_CLOSE); + + org.openide.awt.Mnemonics.setLocalizedText(jLabel1, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.jLabel1.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(imgReadLabel, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.imgReadLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(jLabel2, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.jLabel2.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(dbReadLabel, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.dbReadLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.jLabel4.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(fileReadLabel, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.fileReadLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(jLabel3, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.jLabel3.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(cpuTimeLabel, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.cpuTimeLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(startButton, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.startButton.text")); // NOI18N + startButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + startButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(statusLabel, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.statusLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(jLabel5, org.openide.util.NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.jLabel5.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); + getContentPane().setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(10, 10, 10) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(jLabel1) + .addComponent(jLabel2) + .addComponent(jLabel3)) + .addGap(31, 31, 31) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(fileReadLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 228, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(dbReadLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 237, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(cpuTimeLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 284, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(imgReadLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 237, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGroup(layout.createSequentialGroup() + .addComponent(jLabel4) + .addGap(0, 0, Short.MAX_VALUE)))) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(startButton) + .addComponent(jLabel5)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGroup(layout.createSequentialGroup() + .addComponent(statusLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 508, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGap(0, 0, Short.MAX_VALUE)))) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(jLabel5) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(jLabel3) + .addComponent(cpuTimeLabel)) + .addGap(18, 18, 18) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(imgReadLabel) + .addComponent(jLabel1)) + .addGap(19, 19, 19) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(jLabel4) + .addComponent(fileReadLabel)) + .addGap(21, 21, 21) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(jLabel2) + .addComponent(dbReadLabel)) + .addGap(18, 18, 18) + .addComponent(statusLabel) + .addGap(9, 9, 9) + .addComponent(startButton) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + ); + + pack(); + }// //GEN-END:initComponents + + private void startButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_startButtonActionPerformed + // TODO add your handling code here: + startButton.setEnabled(false); + SwingWorker worker = new PerformanceTestWorker(); + worker.execute(); + + }//GEN-LAST:event_startButtonActionPerformed + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JLabel cpuTimeLabel; + private javax.swing.JLabel dbReadLabel; + private javax.swing.JLabel fileReadLabel; + private javax.swing.JLabel imgReadLabel; + private javax.swing.JLabel jLabel1; + private javax.swing.JLabel jLabel2; + private javax.swing.JLabel jLabel3; + private javax.swing.JLabel jLabel4; + private javax.swing.JLabel jLabel5; + private javax.swing.JButton startButton; + private javax.swing.JLabel statusLabel; + // End of variables declaration//GEN-END:variables + + class PerformanceTestWorker extends SwingWorker { + private long cpuStats; + private long imgReadStats; + private long dbStats; + private long fileReadStats; + + @Override + protected Object doInBackground() throws Exception { + setCpuLabel(""); + setImgLabel(""); + setDbLabel(""); + setFileReadLabel(""); + + doCpuTest(); + doImgTest(); + doFileReadTest(); + doDbTest(); + + return null; + } + + private void setCpuLabel(final String msg) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + cpuTimeLabel.setText(msg); + } + }); + } + + private void setImgLabel(final String msg) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + imgReadLabel.setText(msg); + } + }); + } + + private void setFileReadLabel(final String msg) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + fileReadLabel.setText(msg); + } + }); + } + + private void setDbLabel(final String msg) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + dbReadLabel.setText(msg); + } + }); + } + + private void setStatusMsg(final String msg) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + statusLabel.setText(msg); + } + }); + } + + + private void doCpuTest() { + final String msg = "Running CPU Test"; + + MessageDigest md; + long start = new Date().getTime(); + try { + md = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException ex) { + setCpuLabel("MD5 Algo not found"); + return; + } + + byte[] buf = new byte[256 * 1024]; + long bytesRead = 0; + for (int a = 0; a < 50; a++) { + if (a % 10 == 0) + setStatusMsg(msg + " " + a * 100 / 50 + "%"); + for (byte b = Byte.MIN_VALUE; b < Byte.MAX_VALUE; b++) { + Arrays.fill(buf, b); + md.update(buf); + bytesRead += buf.length; + } + } + md.digest(); + + long end = new Date().getTime(); + cpuStats = (bytesRead / (1024 * 1024)) / ((end - start) / 1000); + + setCpuLabel(cpuStats + " MB hashed / sec"); + setStatusMsg(""); + } + + private void doImgTest() { + imgReadStats = 0; + setStatusMsg("Running Image Reading Test"); + + Case curCase; + try { + curCase = Case.getCurrentCase(); + } + catch (Exception e) { + setImgLabel("Case Not Open"); + setStatusMsg(""); + return; + } + + List images; + try { + images = curCase.getImages(); + } catch (TskCoreException ex) { + setImgLabel("No Images In Case"); + setStatusMsg(""); + return; + } + + if (images.isEmpty()) { + setImgLabel("No Images In Case"); + setStatusMsg(""); + return; + } + + long start = new Date().getTime(); + + Image image = images.get(0); + + byte[] buf = new byte[4096]; + long bytesRead = 0; + + // random starting point to prevent caching from effecting it + Random rand = new Random(); + long curOffset = rand.nextLong(); + if (curOffset < 0) { + curOffset *= -1; + } + curOffset = curOffset % (image.getSize() / 2); + curOffset = 512 * ((curOffset + 511) / 512); + + //long curOffset = 0; + while (bytesRead < 1000 * 1024 * 1024 ) { + long read; + try { + read = image.read(buf, curOffset, buf.length); + } catch (TskCoreException ex) { + break; + } + if (read <= 0) + break; + bytesRead += read; + curOffset += read; + } + long end = new Date().getTime(); + long elapsed = (end - start) / 1000; + if (elapsed > 0) + imgReadStats = (bytesRead / (1024 * 1024)) / elapsed; + else + imgReadStats = 0; + setImgLabel(imgReadStats + " MB read / sec (" + bytesRead + ")"); + setStatusMsg(""); + } + + private void doFileReadTest() { + fileReadStats = 0; + + if (true) { + setFileReadLabel("Skipped"); + } + + setStatusMsg("Running File Reading Test"); + + Case curCase; + try { + curCase = Case.getCurrentCase(); + } + catch (Exception e) { + setFileReadLabel("Case Not Open"); + setStatusMsg(""); + return; + } + + List images; + try { + images = curCase.getImages(); + } catch (TskCoreException ex) { + setFileReadLabel("No Images In Case"); + setStatusMsg(""); + return; + } + + if (images.isEmpty()) { + setFileReadLabel("No Images In Case"); + setStatusMsg(""); + return; + } + + Image image = images.get(0); + File file = new File(image.getPaths()[0]); + if (file.exists() == false) { + setFileReadLabel("Image Path Doesn't Exist"); + setStatusMsg(""); + return; + } + + FileReader fileReader; + try { + fileReader = new FileReader(file); + } catch (FileNotFoundException ex) { + setFileReadLabel("Error making file reader"); + setStatusMsg(""); + return; + } + + long start = new Date().getTime(); + // random starting point to prevent caching from effecting it + // make RandomAccessFile instad + /*Random rand = new Random(); + long curOffset = rand.nextLong(); + if (curOffset < 0) { + curOffset *= -1; + } + curOffset = curOffset % (file.length()); + curOffset = 512 * ((curOffset + 511) / 512); + try { + fileReader.skip(curOffset); + } catch (IOException ex) { + setFileReadLabel("Error seeking: " + curOffset); + return; + }*/ + + char[] buf = new char[4096]; + int bytesRead = 0; + while (bytesRead < 1000 * 1024 * 1024 ) { + long read; + try { + read = fileReader.read(buf, 0, buf.length); + } catch (IOException ex) { + break; + } + if (read <= 0) + break; + bytesRead += read; + } + long end = new Date().getTime(); + long elapsed = (end - start) / 1000; + if (elapsed > 0) + fileReadStats = (bytesRead / (1024 * 1024)) / elapsed; + else + fileReadStats = 0; + setFileReadLabel(fileReadStats + " MB read / sec (" + bytesRead + ")"); + setStatusMsg(""); + } + + private void doDbTest() { + dbStats = 0; + setStatusMsg("Running DB Test"); + + Case curCase; + try { + curCase = Case.getCurrentCase(); + } + catch (Exception e) { + setDbLabel("Case Not Open"); + return; + } + + try { + SleuthkitCase tskCase = curCase.getSleuthkitCase(); + long start = new Date().getTime(); + + List files = tskCase.findAllFilesWhere("obj_id < 50000"); + + long end = new Date().getTime(); + long elapsed = (end - start) / 1000; + if (elapsed > 0) + dbStats = files.size() / elapsed; + else + dbStats = 0; + + setDbLabel(dbStats + " records / sec"); + } catch (TskCoreException ex) { + setDbLabel("Error Performing Query"); + } + + setStatusMsg(""); + } + + @Override + protected void done() { + try { + get(); + } catch (InterruptedException | ExecutionException ex) { + setStatusMsg("Error: " + ex.getMessage()); + } + startButton.setEnabled(true); + } + } +} + diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java new file mode 100755 index 0000000000..0f962e6abf --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java @@ -0,0 +1,46 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.diagnostics; + +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import javax.swing.JDialog; +import org.openide.awt.ActionID; +import org.openide.awt.ActionReference; +import org.openide.awt.ActionRegistration; +import org.openide.util.NbBundle.Messages; + +@ActionID( + category = "Help", + id = "org.sleuthkit.autopsy.diagnostics.PerformancePanelAction" +) +@ActionRegistration( + displayName = "#CTL_PerformancePanelAction" +) +@ActionReference(path = "Menu/Help", position = 1437) +@Messages("CTL_PerformancePanelAction=Performance Diagnostics") +public final class PerformancePanelAction implements ActionListener { + + @Override + public void actionPerformed(ActionEvent e) { + JDialog dialog = new PerformancePanel(); + dialog.setVisible(true); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index e224aa8656..2357bc68ba 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -19,16 +19,13 @@ package org.sleuthkit.autopsy.ingest; import java.text.SimpleDateFormat; -import java.util.Collections; import java.util.Date; -import java.util.Enumeration; import java.util.List; import javax.swing.JDialog; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableColumn; import org.apache.commons.lang3.time.DurationFormatUtils; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.ingest.IngestJob.IngestJobStats; import org.sleuthkit.autopsy.ingest.IngestScheduler.IngestJobSchedulerStats; public class IngestProgressSnapshotPanel extends javax.swing.JPanel { From 88f02e9b251cc6edb73f001c65a9e77fa2b23311 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 13 Aug 2014 06:23:23 -0700 Subject: [PATCH 3/5] Initial commit of module stats --- .../autopsy/ingest/IngestManager.java | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index e9a0fabe2e..5c08a2fb40 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -65,6 +65,7 @@ public class IngestManager { private final ConcurrentHashMap> startIngestJobsTasks = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. + private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); private final Object processedFilesSnapshotLock = new Object(); private ProcessedFilesSnapshot processedFilesSnapshot = new ProcessedFilesSnapshot(); private volatile IngestMessageTopComponent ingestMessageBox; @@ -197,23 +198,52 @@ public class IngestManager { return IngestScheduler.getInstance().ingestJobsAreRunning(); } - void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { + void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource())); } + /** + * Called each time a task proceeds to the next module in the pipeline + * @param task + * @param ingestModuleDisplayName + */ void setIngestTaskProgress(FileIngestTask task, String ingestModuleDisplayName) { - ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile())); + IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); + IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile()); + ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); + + incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } void setIngestTaskProgressCompleted(DataSourceIngestTask task) { ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId())); } + /** + * Called when a file ingest pipeline is complete for a given file + * @param task + */ void setIngestTaskProgressCompleted(FileIngestTask task) { - ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId())); + IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); + IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId()); + ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); synchronized (processedFilesSnapshotLock) { processedFilesSnapshot.incrementProcessedFilesCount(); } + + incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); + } + + private void incrementModuleRunTime(String moduleName, Long duration) { + synchronized (ingestModuleRunTimes) { + Long prevTimeL = ingestModuleRunTimes.get(moduleName); + long prevTime = 0; + if (prevTimeL != null) { + prevTime = prevTimeL; + } + prevTime += duration; + ingestModuleRunTimes.put(moduleName, prevTime); + } } List getIngestThreadActivitySnapshots() { From 6732da6d9a2d64058b7e35b5107a0e286c0ad80a Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 13 Aug 2014 11:28:01 -0700 Subject: [PATCH 4/5] more diagnostics updates. --- .../ShowIngestProgressSnapshotAction.java | 2 +- .../autopsy/ingest/Bundle.properties | 1 - .../autopsy/ingest/Bundle_ja.properties | 147 ++++++++------- .../autopsy/ingest/IngestManager.java | 45 ++++- .../ingest/IngestProgressSnapshotPanel.form | 46 +++-- .../ingest/IngestProgressSnapshotPanel.java | 171 ++++++++++++++---- .../autopsy/ingest/IngestScheduler.java | 41 ++--- 7 files changed, 291 insertions(+), 162 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java b/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java index 82f2b3bfc4..fcedf212b2 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java @@ -38,7 +38,7 @@ import org.sleuthkit.autopsy.ingest.IngestProgressSnapshotDialog; lazy=false ) @ActionReference(path = "Menu/Help", position = 1125) -@Messages("CTL_ShowIngestProgressSnapshotAction=Get Ingest Progress Snapshot") +@Messages("CTL_ShowIngestProgressSnapshotAction=Ingest Status Details") public final class ShowIngestProgressSnapshotAction extends SystemAction implements ActionListener { private static final String ACTION_NAME = NbBundle.getMessage(ShowIngestProgressSnapshotAction.class, "ShowIngestProgressSnapshotAction.actionName.text"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index b750be6dcf..92dd64f8c7 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -75,7 +75,6 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.activity=Activity IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.file=File IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.startTime=Start Time IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.elapsedTime=Elapsed Time (H\:M\:S) -IngestProgressSnapshotPanel.fileProcessedPerSecondLabel.text=Files/Sec: {0} IngestManager.IngestThreadActivitySnapshot.idleThread=IDLE IngestManager.IngestMessage.ErrorMessageLimitReached.subject=Maximum Errors Posted IngestManager.IngestMessage.ErrorMessageLimitReached.msg=Maximum number ({0}) of error and/or warning messages posted. See log for additional errors/warnings (Help -> Open Log Folder). diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties index 4e0a84daf0..ba943a0b71 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties @@ -1,78 +1,77 @@ -CTL_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8 -HINT_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8\u30A6\u30A3\u30F3\u30C9\u30A6 -IngestDialog.closeButton.title=\u9589\u3058\u308B -IngestDialog.startButton.title=\u30B9\u30BF\u30FC\u30C8 -IngestDialog.title.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB -IngestJob.progress.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 -IngestJob.progress.dataSourceIngest.displayName={1}\u306E{0} -IngestJob.progress.fileIngest.displayName={0}\u306E\u30D5\u30A1\u30A4\u30EB\u3092\u89E3\u6790\u4E2D -IngestJobConfigurationPanel.advancedButton.actionCommand=\u30A2\u30C9\u30D0\u30F3\u30B9 -IngestJobConfigurationPanel.advancedButton.text=\u30A2\u30C9\u30D0\u30F3\u30B9 -IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u306E\u51E6\u7406 -IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524A\u9664\u3055\u308C\u305F\u30D5\u30A1\u30A4\u30EB\u7B49\u306E\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u51E6\u7406\u3002\u3088\u308A\u5B8C\u5168\u306A\u7D50\u679C\u304C\u51FA\u307E\u3059\u304C\u3001\u5927\u304D\u3044\u30A4\u30E1\u30FC\u30B8\u3067\u306F\u51E6\u7406\u6642\u9593\u304C\u9577\u304F\u306A\u308B\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 -IngestManager.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC -IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 -IngestManager.StartIngestJobsTask.run.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 -IngestManager.StartIngestJobsTask.run.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u958B\u59CB\u4E2D -IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30BD\u30FC\u30B9\u3001\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u3001\u8A73\u7D30\u304A\u3088\u3073\u30C7\u30FC\u30BF\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 -IngestMessage.exception.srcSubjNotNull.msg=\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 -IngestMessage.exception.typeSrcSubjNotNull.msg=\u30E1\u30C3\u30BB\u30FC\u30B8\u30BF\u30A4\u30D7\u3001\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 -IngestMessage.toString.data.text=\ \u30C7\u30FC\u30BF\uFF1A{0} -IngestMessage.toString.date.text=\ \u65E5\u4ED8\uFF1A{0} -IngestMessage.toString.details.text=\ \u8A73\u7D30\uFF1A{0} -IngestMessage.toString.subject.text=\ \u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\uFF1A{0} -IngestMessage.toString.type.text=\u30BF\u30A4\u30D7\uFF1A{0} -IngestMessageDetailsPanel.copyMenuItem.text=\u30B3\u30D4\u30FC -IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30C6\u30AD\u30B9\u30C8\uFF0Fhtml -IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629E -IngestMessageDetailsPanel.viewArtifactButton.text=\u7D50\u679C\u3078\u79FB\u52D5 -IngestMessageDetailsPanel.viewContentButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u3078\u79FB\u52D5 -IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30D6\u30FC\u30EB\u5024\u3067\u306F\u306A\u3044\u3082\u306E\u306BBooleanRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F\u3002 -IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65E5\u4ED8\u3067\u306F\u306A\u3044\u3082\u306E\u306BDateRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F\u3002 -IngestMessagePanel.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC -IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 -IngestMessagePanel.MsgTableMod.colNames.module=\u30E2\u30B8\u30E5\u30FC\u30EB -IngestMessagePanel.MsgTableMod.colNames.new=\u65B0\u898F\uFF1F -IngestMessagePanel.MsgTableMod.colNames.num=\u756A\u53F7 -IngestMessagePanel.MsgTableMod.colNames.subject=\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8 -IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30BF\u30A4\u30E0\u30B9\u30BF\u30F3\u30D7 -IngestMessagePanel.sortByComboBox.model.priority=\u512A\u5148\u5EA6 +CTL_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8 +HINT_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8\u30a6\u30a3\u30f3\u30c9\u30a6 +IngestDialog.closeButton.title=\u9589\u3058\u308b +IngestDialog.startButton.title=\u30b9\u30bf\u30fc\u30c8 +IngestDialog.title.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb +IngestJob.progress.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09 +IngestJob.progress.dataSourceIngest.displayName={1}\u306e{0} +IngestJob.progress.fileIngest.displayName={0}\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u89e3\u6790\u4e2d +IngestJobConfigurationPanel.advancedButton.actionCommand=\u30a2\u30c9\u30d0\u30f3\u30b9 +IngestJobConfigurationPanel.advancedButton.text=\u30a2\u30c9\u30d0\u30f3\u30b9 +IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u306e\u51e6\u7406 +IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524a\u9664\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb\u7b49\u306e\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u3092\u51e6\u7406\u3002\u3088\u308a\u5b8c\u5168\u306a\u7d50\u679c\u304c\u51fa\u307e\u3059\u304c\u3001\u5927\u304d\u3044\u30a4\u30e1\u30fc\u30b8\u3067\u306f\u51e6\u7406\u6642\u9593\u304c\u9577\u304f\u306a\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 +IngestManager.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc +IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 +IngestManager.StartIngestJobsTask.run.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09 +IngestManager.StartIngestJobsTask.run.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u958b\u59cb\u4e2d +IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30bd\u30fc\u30b9\u3001\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u3001\u8a73\u7d30\u304a\u3088\u3073\u30c7\u30fc\u30bf\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093 +IngestMessage.exception.srcSubjNotNull.msg=\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093 +IngestMessage.exception.typeSrcSubjNotNull.msg=\u30e1\u30c3\u30bb\u30fc\u30b8\u30bf\u30a4\u30d7\u3001\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093 +IngestMessage.toString.data.text=\ \u30c7\u30fc\u30bf\uff1a{0} +IngestMessage.toString.date.text=\ \u65e5\u4ed8\uff1a{0} +IngestMessage.toString.details.text=\ \u8a73\u7d30\uff1a{0} +IngestMessage.toString.subject.text=\ \u30b5\u30d6\u30b8\u30a7\u30af\u30c8\uff1a{0} +IngestMessage.toString.type.text=\u30bf\u30a4\u30d7\uff1a{0} +IngestMessageDetailsPanel.copyMenuItem.text=\u30b3\u30d4\u30fc +IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30c6\u30ad\u30b9\u30c8\uff0fhtml +IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629e +IngestMessageDetailsPanel.viewArtifactButton.text=\u7d50\u679c\u3078\u79fb\u52d5 +IngestMessageDetailsPanel.viewContentButton.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3078\u79fb\u52d5 +IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30d6\u30fc\u30eb\u5024\u3067\u306f\u306a\u3044\u3082\u306e\u306bBooleanRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f\u3002 +IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65e5\u4ed8\u3067\u306f\u306a\u3044\u3082\u306e\u306bDateRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f\u3002 +IngestMessagePanel.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc +IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 +IngestMessagePanel.MsgTableMod.colNames.module=\u30e2\u30b8\u30e5\u30fc\u30eb +IngestMessagePanel.MsgTableMod.colNames.new=\u65b0\u898f\uff1f +IngestMessagePanel.MsgTableMod.colNames.num=\u756a\u53f7 +IngestMessagePanel.MsgTableMod.colNames.subject=\u30b5\u30d6\u30b8\u30a7\u30af\u30c8 +IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30bf\u30a4\u30e0\u30b9\u30bf\u30f3\u30d7 +IngestMessagePanel.sortByComboBox.model.priority=\u512a\u5148\u5ea6 IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593 -IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uFF08\u6642\u7CFB\u5217\uFF09\u307E\u305F\u306F\u30E1\u30C3\u30BB\u30FC\u30B8\u306E\u512A\u5148\u5EA6\u3067\u30BD\u30FC\u30C8 -IngestMessagePanel.sortByLabel.text=\u4E0B\u8A18\u3067\u30BD\u30FC\u30C8\uFF1A -IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8A08\uFF1A +IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uff08\u6642\u7cfb\u5217\uff09\u307e\u305f\u306f\u30e1\u30c3\u30bb\u30fc\u30b8\u306e\u512a\u5148\u5ea6\u3067\u30bd\u30fc\u30c8 +IngestMessagePanel.sortByLabel.text=\u4e0b\u8a18\u3067\u30bd\u30fc\u30c8\uff1a +IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8a08\uff1a IngestMessagePanel.totalMessagesNameVal.text=- -IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30E6\u30CB\u30FC\u30AF\uFF1A +IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30e6\u30cb\u30fc\u30af\uff1a IngestMessagePanel.totalUniqueMessagesNameVal.text=- -IngestMessagesToolbar.customizeButton.toolTipText=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E1\u30C3\u30BB\u30FC\u30B8 -IngestMessageTopComponent.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 -IngestMessageTopComponent.displayReport.option.GenRpt=\u30EC\u30DD\u30FC\u30C8\u751F\u6210 +IngestMessagesToolbar.customizeButton.toolTipText=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e1\u30c3\u30bb\u30fc\u30b8 +IngestMessageTopComponent.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9 +IngestMessageTopComponent.displayReport.option.GenRpt=\u30ec\u30dd\u30fc\u30c8\u751f\u6210 IngestMessageTopComponent.displayReport.option.OK=OK -IngestMessageTopComponent.initComponents.name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 -IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30EC\u30DD\u30FC\u30C8 -IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30C7\u30A3\u30B9\u30AF{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3\u306E\u305F\u3081\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u4E2D\u6B62\u3057\u307E\u3059\u3002\n\u30B1\u30FC\u30B9\u30C9\u30E9\u30A4\u30D6\u306B\u6700\u4F4E1GB\u306E\u7A7A\u304D\u9818\u57DF\u304C\u3042\u308B\u306E\u3092\u78BA\u8A8D\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u518D\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 -IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u304C\u4E2D\u6B62\u3055\u308C\u307E\u3057\u305F\u30FC{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3 -OpenIDE-Module-Name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 -IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\n\u30A8\u30E9\u30FC\uFF1A\n{0} -IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uFF11\u3064\u307E\u305F\u306F\u8907\u6570\u306E\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u30B9\u30BF\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u306F\u30AD\u30E3\u30F3\u30BB\u30EB\u3055\u308C\u307E\u3057\u305F\u3002 -IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305F\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u7121\u52B9\u5316\u3059\u308B\u304B\u30A8\u30E9\u30FC\u3092\u89E3\u6C7A\u3057\u3001\u305D\u306E\u5F8C\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u53F3\u30AF\u30EA\u30C3\u30AF\u3057\u3001\n\u300C\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u5B9F\u884C\u300D\u3092\u9078\u629E\u3057\u3066\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u30EA\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 -IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u5931\u6557 -IngestJobConfigurator.createModuleSettingsFolderForContext.exception.msg=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u8A2D\u5B9A\u30D5\u30A9\u30EB\u30C0\u306E\u4F5C\u6210\u306B\u5931\u6557\u3057\u307E\u3057\u305F\u3002\u8A2D\u5B9A\u3092\u4FDD\u5B58\u3067\u304D\u307E\u305B\u3093\u3002 -IngestJobConfigurator.createModuleSettingsFolderForContext.exception.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u521D\u671F\u5316\u306E\u5931\u6557 -IngestJobConfigurator.saveJobSettings.usermsg={0}\u30E2\u30B8\u30E5\u30FC\u30EB\u306E\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u8A2D\u5B9A\u306E\u4FDD\u5B58\u306B\u5931\u6557\u3057\u307E\u3057\u305F\u3002 -IngestJobConfigurator.saveJobSettings.usermsg.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u8A2D\u5B9A -IngestJob.progress.dataSourceIngest.initialDisplayName={0}\u3092\u89E3\u6790\u4E2D -IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.dataSource=\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9 -IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.elapsedTime=\u7D4C\u904E\u6642\u9593\uFF08\u6642\uFF1A\u5206\uFF1A\u79D2\uFF09 -IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.file=\u30D5\u30A1\u30A4\u30EB -IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.startTime=\u958B\u59CB\u6642\u9593 -IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.threadID=\u30B9\u30EC\u30C3\u30C9ID -IngestManager.IngestMessage.ErrorMessageLimitReached.msg=\u6700\u5927\u6570({0})\u306E\u30A8\u30E9\u30FC\u304A\u3088\u3073\u307E\u305F\u306F\u8B66\u544A\u30E1\u30C3\u30BB\u30FC\u30B8\u304C\u63B2\u8F09\u3055\u308C\u307E\u3057\u305F\u3002\u3055\u3089\u306A\u308B\u30A8\u30E9\u30FC\uFF0F\u8B66\u544A\u306F\u30ED\u30B0\u3092\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\uFF08\u30D8\u30EB\u30D7->\u30ED\u30B0\u30D5\u30A9\u30EB\u30C0\u30FC\u3092\u958B\u304F\uFF09 -IngestManager.IngestMessage.ErrorMessageLimitReached.subject=\u6700\u5927\u6570\u306E\u30A8\u30E9\u30FC\u304C\u63B2\u8F09\u3055\u308C\u307E\u3057\u305F -IngestManager.IngestMessage.ErrorMessageLimitReached.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30DE\u30CD\u30B8\u30E3\u30FC -IngestManager.IngestThreadActivitySnapshot.idleThread=\u30A2\u30A4\u30C9\u30EB -IngestProgressSnapshotDialog.title.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30D7\u30ED\u30B0\u30EC\u30B9\u30FB\u30B9\u30CA\u30C3\u30D7\u30B7\u30E7\u30C3\u30C8 -IngestProgressSnapshotPanel.closeButton.text=\u9589\u3058\u308B -IngestProgressSnapshotPanel.fileProcessedPerSecondLabel.text=\u30D5\u30A1\u30A4\u30EB\uFF0F\u79D2\uFF1A{0} -IngestProgressSnapshotPanel.refreshButton.text=\u30EA\u30D5\u30EC\u30C3\u30B7\u30E5 -IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.activity=\u30A2\u30AF\u30C6\u30A3\u30D3\u30C6\u30A3 +IngestMessageTopComponent.initComponents.name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9 +IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30ec\u30dd\u30fc\u30c8 +IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3\u306e\u305f\u3081\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u4e2d\u6b62\u3057\u307e\u3059\u3002\n\u30b1\u30fc\u30b9\u30c9\u30e9\u30a4\u30d6\u306b\u6700\u4f4e1GB\u306e\u7a7a\u304d\u9818\u57df\u304c\u3042\u308b\u306e\u3092\u78ba\u8a8d\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u518d\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002 +IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3 +OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8 +IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\n\u30a8\u30e9\u30fc\uff1a\n{0} +IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uff11\u3064\u307e\u305f\u306f\u8907\u6570\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u30b9\u30bf\u30fc\u30c8\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\u3002\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u306f\u30ad\u30e3\u30f3\u30bb\u30eb\u3055\u308c\u307e\u3057\u305f\u3002 +IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305f\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u7121\u52b9\u5316\u3059\u308b\u304b\u30a8\u30e9\u30fc\u3092\u89e3\u6c7a\u3057\u3001\u305d\u306e\u5f8c\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u53f3\u30af\u30ea\u30c3\u30af\u3057\u3001\n\u300c\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u5b9f\u884c\u300d\u3092\u9078\u629e\u3057\u3066\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u30ea\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002 +IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u5931\u6557 +IngestJobConfigurator.createModuleSettingsFolderForContext.exception.msg=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u8a2d\u5b9a\u30d5\u30a9\u30eb\u30c0\u306e\u4f5c\u6210\u306b\u5931\u6557\u3057\u307e\u3057\u305f\u3002\u8a2d\u5b9a\u3092\u4fdd\u5b58\u3067\u304d\u307e\u305b\u3093\u3002 +IngestJobConfigurator.createModuleSettingsFolderForContext.exception.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u521d\u671f\u5316\u306e\u5931\u6557 +IngestJobConfigurator.saveJobSettings.usermsg={0}\u30e2\u30b8\u30e5\u30fc\u30eb\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u8a2d\u5b9a\u306e\u4fdd\u5b58\u306b\u5931\u6557\u3057\u307e\u3057\u305f\u3002 +IngestJobConfigurator.saveJobSettings.usermsg.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u8a2d\u5b9a +IngestJob.progress.dataSourceIngest.initialDisplayName={0}\u3092\u89e3\u6790\u4e2d +IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.dataSource=\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9 +IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.elapsedTime=\u7d4c\u904e\u6642\u9593\uff08\u6642\uff1a\u5206\uff1a\u79d2\uff09 +IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.file=\u30d5\u30a1\u30a4\u30eb +IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.startTime=\u958b\u59cb\u6642\u9593 +IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.threadID=\u30b9\u30ec\u30c3\u30c9ID +IngestManager.IngestMessage.ErrorMessageLimitReached.msg=\u6700\u5927\u6570({0})\u306e\u30a8\u30e9\u30fc\u304a\u3088\u3073\u307e\u305f\u306f\u8b66\u544a\u30e1\u30c3\u30bb\u30fc\u30b8\u304c\u63b2\u8f09\u3055\u308c\u307e\u3057\u305f\u3002\u3055\u3089\u306a\u308b\u30a8\u30e9\u30fc\uff0f\u8b66\u544a\u306f\u30ed\u30b0\u3092\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\uff08\u30d8\u30eb\u30d7->\u30ed\u30b0\u30d5\u30a9\u30eb\u30c0\u30fc\u3092\u958b\u304f\uff09 +IngestManager.IngestMessage.ErrorMessageLimitReached.subject=\u6700\u5927\u6570\u306e\u30a8\u30e9\u30fc\u304c\u63b2\u8f09\u3055\u308c\u307e\u3057\u305f +IngestManager.IngestMessage.ErrorMessageLimitReached.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30de\u30cd\u30b8\u30e3\u30fc +IngestManager.IngestThreadActivitySnapshot.idleThread=\u30a2\u30a4\u30c9\u30eb +IngestProgressSnapshotDialog.title.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30d7\u30ed\u30b0\u30ec\u30b9\u30fb\u30b9\u30ca\u30c3\u30d7\u30b7\u30e7\u30c3\u30c8 +IngestProgressSnapshotPanel.closeButton.text=\u9589\u3058\u308b +IngestProgressSnapshotPanel.refreshButton.text=\u30ea\u30d5\u30ec\u30c3\u30b7\u30e5 +IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.activity=\u30a2\u30af\u30c6\u30a3\u30d3\u30c6\u30a3 diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 5c08a2fb40..b629c790d9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -23,7 +23,9 @@ import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -198,12 +200,18 @@ public class IngestManager { return IngestScheduler.getInstance().ingestJobsAreRunning(); } + + /** + * Called each time a module in a data source pipeline starts + * @param task + * @param ingestModuleDisplayName + */ void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource())); } /** - * Called each time a task proceeds to the next module in the pipeline + * Called each time a module in a file ingest pipeline starts * @param task * @param ingestModuleDisplayName */ @@ -215,6 +223,10 @@ public class IngestManager { incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } + /** + * Called each time a data source ingest task completes + * @param task + */ void setIngestTaskProgressCompleted(DataSourceIngestTask task) { ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId())); } @@ -234,7 +246,15 @@ public class IngestManager { incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } + /** + * Internal method to update the times associated with each module. + * @param moduleName + * @param duration + */ private void incrementModuleRunTime(String moduleName, Long duration) { + if (moduleName.equals("IDLE")) + return; + synchronized (ingestModuleRunTimes) { Long prevTimeL = ingestModuleRunTimes.get(moduleName); long prevTime = 0; @@ -245,19 +265,26 @@ public class IngestManager { ingestModuleRunTimes.put(moduleName, prevTime); } } + + /** + * Return the list of run times for each module + * @return Map of module name to run time (in milliseconds) + */ + Map getModuleRunTimes() { + synchronized (ingestModuleRunTimes) { + Map times = new HashMap<>(ingestModuleRunTimes); + return times; + } + } + /** + * Get the stats on current state of each thread + * @return + */ List getIngestThreadActivitySnapshots() { return new ArrayList(ingestThreadActivitySnapshots.values()); } - ProcessedFilesSnapshot getProcessedFilesSnapshot() { - ProcessedFilesSnapshot snapshot; - synchronized (processedFilesSnapshotLock) { - snapshot = processedFilesSnapshot; - processedFilesSnapshot = new ProcessedFilesSnapshot(); - } - return snapshot; - } public void cancelAllIngestJobs() { // Stop creating new ingest jobs. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form index cf8122d2a3..5c0c3dbc16 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.form @@ -21,13 +21,13 @@ - - + - + + @@ -37,14 +37,15 @@ - + - + + + - @@ -74,14 +75,14 @@ - + - +
@@ -116,12 +117,27 @@ - - - - - - - + + + + + + + + + + +
+ + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 2357bc68ba..dff5c2c856 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -19,8 +19,11 @@ package org.sleuthkit.autopsy.ingest; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import javax.swing.JDialog; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableColumn; @@ -33,18 +36,21 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final JDialog parent; private final IngestThreadActivitySnapshotsTableModel threadActivityTableModel; private final IngestJobTableModel jobTableModel; + private final ModuleTableModel moduleTableModel; IngestProgressSnapshotPanel(JDialog parent) { this.parent = parent; threadActivityTableModel = new IngestThreadActivitySnapshotsTableModel(); jobTableModel = new IngestJobTableModel(); + moduleTableModel = new ModuleTableModel(); initComponents(); customizeComponents(); } private void customizeComponents() { threadActivitySnapshotsTable.setModel(threadActivityTableModel); - jobsTable.setModel(jobTableModel); + jobTable.setModel(jobTableModel); + moduleTable.setModel(moduleTableModel); int width = snapshotsScrollPane.getPreferredSize().width; for (int i = 0; i < threadActivitySnapshotsTable.getColumnCount(); ++i) { @@ -72,9 +78,6 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { } threadActivitySnapshotsTable.setFillsViewportHeight(true); - - fileProcessedPerSecondLabel.setText(NbBundle.getMessage(this.getClass(), - "IngestProgressSnapshotPanel.fileProcessedPerSecondLabel.text", 0)); } private class IngestThreadActivitySnapshotsTableModel extends AbstractTableModel { @@ -157,7 +160,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private class IngestJobTableModel extends AbstractTableModel { private final String[] columnNames = {"Job ID", - "Data Source", "Start", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; + "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; private List schedStats; private IngestJobTableModel() { @@ -197,24 +200,27 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { break; case 2: SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); - cellValue = dateFormat.format(new Date(schedStat.ingestJobStats.getStartTime())); + cellValue = dateFormat.format(new Date(schedStat.getIngestJobStats().getStartTime())); break; case 3: - cellValue = schedStat.ingestJobStats.getSpeed(); + cellValue = schedStat.getIngestJobStats().getFilesProcessed(); break; case 4: - cellValue = schedStat.getRunningListSize(); + cellValue = schedStat.getIngestJobStats().getSpeed(); break; case 5: - cellValue = schedStat.getFileQueueSize(); + cellValue = schedStat.getRunningListSize(); break; case 6: - cellValue = schedStat.getDirQueueSize(); + cellValue = schedStat.getFileQueueSize(); break; case 7: - cellValue = schedStat.getRootQueueSize(); + cellValue = schedStat.getDirQueueSize(); break; case 8: + cellValue = schedStat.getRootQueueSize(); + break; + case 9: cellValue = schedStat.getDsQueueSize(); break; default: @@ -224,6 +230,99 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { return cellValue; } } + + private class ModuleTableModel extends AbstractTableModel { + + private class ModuleStats implements Comparable { + private final String name; + private final long duration; + ModuleStats(String name, long duration) { + this.name = name; + this.duration = duration; + } + + /** + * @return the name + */ + protected String getName() { + return name; + } + + /** + * @return the duration + */ + protected long getDuration() { + return duration; + } + + @Override + public int compareTo(ModuleStats o) { + if (duration > o.getDuration()) { + return -1; + } + else if (duration == o.getDuration()) { + return 0; + } + else { + return 1; + } + } + + } + private final String[] columnNames = {"Module", "Duration"}; + private final List moduleStats = new ArrayList<>(); + private long totalTime; + + private ModuleTableModel() { + refresh(); + } + + private void refresh() { + MapmoduleStatMap = IngestManager.getInstance().getModuleRunTimes(); + moduleStats.clear(); + totalTime = 0; + for (String k : moduleStatMap.keySet()) { + moduleStats.add(new ModuleStats(k, moduleStatMap.get(k))); + totalTime += moduleStatMap.get(k); + } + Collections.sort(moduleStats); + fireTableDataChanged(); + } + + @Override + public int getRowCount() { + return moduleStats.size(); + } + + @Override + public int getColumnCount() { + return columnNames.length; + } + + @Override + public String getColumnName(int col) { + return columnNames[col]; + } + + @Override + public Object getValueAt(int rowIndex, int columnIndex) { + ModuleStats moduleStat = moduleStats.get(rowIndex); + Object cellValue; + switch (columnIndex) { + case 0: + cellValue = moduleStat.getName(); + break; + case 1: + cellValue = DurationFormatUtils.formatDurationHMS(moduleStat.getDuration()) + " (" + (moduleStat.getDuration() * 100) / totalTime + "%)"; + break; + + default: + cellValue = null; + break; + } + return cellValue; + } + } /** * This method is called from within the constructor to initialize the form. @@ -236,11 +335,12 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { snapshotsScrollPane = new javax.swing.JScrollPane(); threadActivitySnapshotsTable = new javax.swing.JTable(); - jobsScrollPane = new javax.swing.JScrollPane(); - jobsTable = new javax.swing.JTable(); + jobScrollPane = new javax.swing.JScrollPane(); + jobTable = new javax.swing.JTable(); refreshButton = new javax.swing.JButton(); closeButton = new javax.swing.JButton(); - fileProcessedPerSecondLabel = new javax.swing.JLabel(); + moduleScrollPane = new javax.swing.JScrollPane(); + moduleTable = new javax.swing.JTable(); threadActivitySnapshotsTable.setModel(new javax.swing.table.DefaultTableModel( new Object [][] { @@ -252,7 +352,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { )); snapshotsScrollPane.setViewportView(threadActivitySnapshotsTable); - jobsTable.setModel(new javax.swing.table.DefaultTableModel( + jobTable.setModel(new javax.swing.table.DefaultTableModel( new Object [][] { }, @@ -260,7 +360,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { } )); - jobsScrollPane.setViewportView(jobsTable); + jobScrollPane.setViewportView(jobTable); org.openide.awt.Mnemonics.setLocalizedText(refreshButton, org.openide.util.NbBundle.getMessage(IngestProgressSnapshotPanel.class, "IngestProgressSnapshotPanel.refreshButton.text")); // NOI18N refreshButton.addActionListener(new java.awt.event.ActionListener() { @@ -276,7 +376,15 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { } }); - org.openide.awt.Mnemonics.setLocalizedText(fileProcessedPerSecondLabel, org.openide.util.NbBundle.getMessage(IngestProgressSnapshotPanel.class, "IngestProgressSnapshotPanel.fileProcessedPerSecondLabel.text")); // NOI18N + moduleTable.setModel(new javax.swing.table.DefaultTableModel( + new Object [][] { + + }, + new String [] { + + } + )); + moduleScrollPane.setViewportView(moduleTable); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -287,12 +395,12 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addComponent(snapshotsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 881, Short.MAX_VALUE) .addGroup(layout.createSequentialGroup() - .addComponent(fileProcessedPerSecondLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 173, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGap(0, 0, Short.MAX_VALUE) .addComponent(refreshButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(closeButton)) - .addComponent(jobsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 881, Short.MAX_VALUE)) + .addComponent(jobScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 881, Short.MAX_VALUE) + .addComponent(moduleScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 881, Short.MAX_VALUE)) .addContainerGap()) ); @@ -302,14 +410,15 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addContainerGap() - .addComponent(snapshotsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 155, Short.MAX_VALUE) + .addComponent(snapshotsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 102, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(jobsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 155, Short.MAX_VALUE) + .addComponent(jobScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 102, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(moduleScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 100, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(refreshButton) - .addComponent(closeButton) - .addComponent(fileProcessedPerSecondLabel)) + .addComponent(closeButton)) .addContainerGap()) ); @@ -324,18 +433,14 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private void refreshButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_refreshButtonActionPerformed threadActivityTableModel.refresh(); jobTableModel.refresh(); - IngestManager.ProcessedFilesSnapshot snapshot = IngestManager.getInstance().getProcessedFilesSnapshot(); - Date now = new Date(); - long elapsedTime = now.getTime() - snapshot.getStartTime().getTime(); - double filesPerSecond = (double) snapshot.getProcessedFilesCount() / (double) elapsedTime * 1000.0; - fileProcessedPerSecondLabel.setText(NbBundle.getMessage(this.getClass(), - "IngestProgressSnapshotPanel.fileProcessedPerSecondLabel.text", filesPerSecond)); + moduleTableModel.refresh(); }//GEN-LAST:event_refreshButtonActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton closeButton; - private javax.swing.JLabel fileProcessedPerSecondLabel; - private javax.swing.JScrollPane jobsScrollPane; - private javax.swing.JTable jobsTable; + private javax.swing.JScrollPane jobScrollPane; + private javax.swing.JTable jobTable; + private javax.swing.JScrollPane moduleScrollPane; + private javax.swing.JTable moduleTable; private javax.swing.JButton refreshButton; private javax.swing.JScrollPane snapshotsScrollPane; private javax.swing.JTable threadActivitySnapshotsTable; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 13a599c6a1..021f797d1b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -595,14 +595,14 @@ final class IngestScheduler { * Stores basic stats for a given job */ class IngestJobSchedulerStats { - IngestJobStats ingestJobStats; - private long jobId; - private String dataSource; - private long rootQueueSize; - private long dirQueueSize; - private long fileQueueSize; - private long dsQueueSize; - private long runningListSize; + private final IngestJobStats ingestJobStats; + private final long jobId; + private final String dataSource; + private final long rootQueueSize; + private final long dirQueueSize; + private final long fileQueueSize; + private final long dsQueueSize; + private final long runningListSize; IngestJobSchedulerStats(IngestJob job) { ingestJobStats = job.getStats(); @@ -615,54 +615,37 @@ final class IngestScheduler { runningListSize = countJobsInCollection(tasksInProgressAndPending, jobId) - fileQueueSize - dsQueueSize; } - /** - * @return the jobId - */ protected long getJobId() { return jobId; } - /** - * @return the dataSource - */ protected String getDataSource() { return dataSource; } - /** - * @return the rootQueueSize - */ protected long getRootQueueSize() { return rootQueueSize; } - /** - * @return the dirQueueSize - */ protected long getDirQueueSize() { return dirQueueSize; } - /** - * @return the fileQueueSize - */ protected long getFileQueueSize() { return fileQueueSize; } - /** - * @return the dsQueueSize - */ protected long getDsQueueSize() { return dsQueueSize; } - /** - * @return the runningListSize - */ protected long getRunningListSize() { return runningListSize; } + + protected IngestJobStats getIngestJobStats() { + return ingestJobStats; + } } /** From 5f74de3cf81901c92168aeef6f2ac02b7b70c665 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 13 Aug 2014 11:36:01 -0700 Subject: [PATCH 5/5] removed author tag --- .../org/sleuthkit/autopsy/diagnostics/PerformancePanel.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java index 55ea404b2e..6361de13df 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java @@ -34,7 +34,6 @@ import java.util.concurrent.ExecutionException; import javax.swing.JFrame; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; -import org.openide.util.Exceptions; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.datamodel.AbstractFile; @@ -42,10 +41,6 @@ import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; -/** - * - * @author brianc - */ public class PerformancePanel extends javax.swing.JDialog { /**