From 446447fa8beab1143d0702924596d43878078fc4 Mon Sep 17 00:00:00 2001 From: Devin148 Date: Mon, 10 Dec 2012 11:10:19 -0500 Subject: [PATCH 1/6] Simplify method name --- Core/src/org/sleuthkit/autopsy/report/ReportBodyFile.java | 2 +- Core/src/org/sleuthkit/autopsy/report/ReportHTML.java | 2 +- Core/src/org/sleuthkit/autopsy/report/ReportModule.java | 2 +- Core/src/org/sleuthkit/autopsy/report/ReportXLS.java | 2 +- Core/src/org/sleuthkit/autopsy/report/ReportXML.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportBodyFile.java b/Core/src/org/sleuthkit/autopsy/report/ReportBodyFile.java index 1b54485211..6cb5e022a8 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportBodyFile.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportBodyFile.java @@ -242,7 +242,7 @@ public class ReportBodyFile implements ReportModule { } @Override - public String getReportTypeDescription() { + public String getDescription() { String desc = "Body file format report with MAC times for every file, that can be used for a timeline view."; return desc; } diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index f1b2ed79eb..461475c021 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -1373,7 +1373,7 @@ public class ReportHTML implements ReportModule { } @Override - public String getReportTypeDescription() { + public String getDescription() { String desc = "This is an html formatted report that is meant to be viewed in a modern browser."; return desc; } diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportModule.java b/Core/src/org/sleuthkit/autopsy/report/ReportModule.java index 5e6bbfd1e9..9a3e77ef75 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportModule.java @@ -75,7 +75,7 @@ public interface ReportModule { * module generates * @return user-friendly report description */ - public String getReportTypeDescription(); + public String getDescription(); /** * Calls to the report module to execute a method to display the report that diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportXLS.java b/Core/src/org/sleuthkit/autopsy/report/ReportXLS.java index b09fc419bf..22e19bdd3d 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportXLS.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportXLS.java @@ -575,7 +575,7 @@ public class ReportXLS implements ReportModule { } @Override - public String getReportTypeDescription() { + public String getDescription() { String desc = "This is an xls formatted report that is meant to be viewed in Excel."; return desc; } diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportXML.java b/Core/src/org/sleuthkit/autopsy/report/ReportXML.java index b11303b599..bcbb42ed46 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportXML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportXML.java @@ -305,7 +305,7 @@ public class ReportXML implements ReportModule { } @Override - public String getReportTypeDescription() { + public String getDescription() { String desc = "This is an xml formatted report that is meant to be viewed in a modern browser."; return desc; } From f93a1b0443f08d6710b9a117751e6a34e0698faf Mon Sep 17 00:00:00 2001 From: Devin148 Date: Mon, 10 Dec 2012 11:12:31 -0500 Subject: [PATCH 2/6] Publicize tag API for use with reporting --- Core/src/org/sleuthkit/autopsy/datamodel/Tags.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java index 330feb8acf..d1673a4d09 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java @@ -122,7 +122,7 @@ public class Tags { * Get a list of all the tag names. * @return a list of all tag names. */ - static String[] getTagNames() { + public static String[] getTagNames() { Set names = new HashSet(); //List names = new ArrayList(); try { @@ -153,7 +153,7 @@ public class Tags { * @param name of the requested tags * @return a list of all tag artifacts with the given name */ - static List getTagsByName(String name) { + public static List getTagsByName(String name) { try { Case currentCase = Case.getCurrentCase(); SleuthkitCase skCase = currentCase.getSleuthkitCase(); From 246341e6f9e44873e9eda5ecc859967e2a8bf942 Mon Sep 17 00:00:00 2001 From: Devin148 Date: Fri, 21 Dec 2012 12:41:27 -0500 Subject: [PATCH 3/6] Change Tag name finding to custom query for performance --- .../org/sleuthkit/autopsy/datamodel/Tags.java | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java index d1673a4d09..5ebbd2585a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java @@ -18,16 +18,17 @@ */ package org.sleuthkit.autopsy.datamodel; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.logging.Level; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -120,32 +121,35 @@ public class Tags { /** * Get a list of all the tag names. + * Uses a custom query for speed when dealing with thousands of Tags. * @return a list of all tag names. */ - public static String[] getTagNames() { - Set names = new HashSet(); - //List names = new ArrayList(); + public static List getTagNames() { + Case currentCase = Case.getCurrentCase(); + SleuthkitCase skCase = currentCase.getSleuthkitCase(); + List names = new ArrayList(); + ResultSet rs = null; try { - Case currentCase = Case.getCurrentCase(); - SleuthkitCase skCase = currentCase.getSleuthkitCase(); - List fileTags = skCase.getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_TAG_FILE); - List artifactTags = skCase.getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_TAG_ARTIFACT); - fileTags.addAll(artifactTags); - - for(BlackboardArtifact artifact : fileTags) { - List attributes = artifact.getAttributes(); - for(BlackboardAttribute att : attributes) { - if(att.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TAG_NAME.getTypeID()) { - names.add(att.getValueString()); - break; - } + rs = skCase.runQuery("SELECT value_text" + + " FROM blackboard_attributes" + + " WHERE attribute_type_id = " + ATTRIBUTE_TYPE.TSK_TAG_NAME.getTypeID() + + " GROUP BY value_text" + + " ORDER BY value_text"); + while(rs.next()) { + names.add(rs.getString("value_text")); + } + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to query the blackboard for tag names."); + } finally { + if (rs != null) { + try { + skCase.closeRunQuery(rs); + } catch (SQLException ex) { } } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Failed to get list of artifacts from the case."); } - return names.toArray(new String[0]); + return names; } /** From 7d85c833e94432bf676b765da2a1e1bfb08d238e Mon Sep 17 00:00:00 2001 From: Devin148 Date: Fri, 21 Dec 2012 13:57:25 -0500 Subject: [PATCH 4/6] Remake reporting UI and functionality - Add TableReportModule and GeneralReportModule - Allow a custom configuration panel for reports - Add cancelation support per-report - Reduce memory usage through one iteration for TableReportModules - Add reporting on tagged results - Optimize artifact sorting --- Core/src/org/sleuthkit/autopsy/core/layer.xml | 29 +- .../report/ArtifactSelectionDialog.form | 128 ++ .../report/ArtifactSelectionDialog.java | 281 +++ .../autopsy/report/BrowserControl.java | 64 - .../autopsy/report/Bundle.properties | 48 +- .../DefaultReportConfigurationPanel.form | 48 + .../DefaultReportConfigurationPanel.java | 67 + .../autopsy/report/GeneralReportModule.java | 42 + .../org/sleuthkit/autopsy/report/Report.java | 305 ---- .../autopsy/report/ReportAction.java | 263 --- .../autopsy/report/ReportBodyFile.java | 188 +- .../autopsy/report/ReportConfiguration.java | 141 -- .../sleuthkit/autopsy/report/ReportExcel.java | 251 +++ .../autopsy/report/ReportFilter.form | 135 -- .../autopsy/report/ReportFilter.java | 289 --- .../autopsy/report/ReportFilterAction.java | 84 - .../sleuthkit/autopsy/report/ReportGen.java | 55 - ...tPanel.form => ReportGenerationPanel.form} | 120 +- .../autopsy/report/ReportGenerationPanel.java | 237 +++ .../autopsy/report/ReportGenerator.java | 950 ++++++++++ .../sleuthkit/autopsy/report/ReportHTML.java | 1564 +++++------------ .../autopsy/report/ReportModule.java | 55 +- .../autopsy/report/ReportModuleException.java | 35 - .../sleuthkit/autopsy/report/ReportPanel.java | 282 --- .../autopsy/report/ReportPanelAction.java | 163 -- .../autopsy/report/ReportProgressPanel.form | 128 ++ .../autopsy/report/ReportProgressPanel.java | 396 +++++ .../sleuthkit/autopsy/report/ReportUtils.java | 61 - .../autopsy/report/ReportVisualPanel1.form | 139 ++ .../autopsy/report/ReportVisualPanel1.java | 298 ++++ .../autopsy/report/ReportVisualPanel2.form | 160 ++ .../autopsy/report/ReportVisualPanel2.java | 346 ++++ .../autopsy/report/ReportWizardAction.java | 163 ++ .../autopsy/report/ReportWizardIterator.java | 99 ++ .../autopsy/report/ReportWizardPanel1.java | 108 ++ .../autopsy/report/ReportWizardPanel2.java | 88 + .../sleuthkit/autopsy/report/ReportXLS.java | 592 ------- .../sleuthkit/autopsy/report/ReportXML.java | 314 ---- .../autopsy/report/TableReportModule.java | 122 ++ .../autopsy/report/{ => images}/bookmarks.png | Bin .../{ => images}/btn_icon_generate_report.png | Bin .../autopsy/report/{ => images}/cookies.png | Bin .../autopsy/report/{ => images}/devices.png | Bin .../autopsy/report/{ => images}/downloads.png | Bin .../autopsy/report/{ => images}/exif.png | Bin .../autopsy/report/{ => images}/favicon.ico | Bin .../autopsy/report/{ => images}/hash.png | Bin .../autopsy/report/{ => images}/history.png | Bin .../autopsy/report/{ => images}/installed.png | Bin .../autopsy/report/{ => images}/keywords.png | Bin .../autopsy/report/{ => images}/logo.png | Bin .../autopsy/report/{ => images}/recent.png | Bin .../autopsy/report/images/report_cancel.png | Bin 0 -> 877 bytes .../report/images/report_cancel_hover.png | Bin 0 -> 3679 bytes .../autopsy/report/images/report_complete.png | Bin 0 -> 634 bytes .../autopsy/report/images/report_loading.png | Bin 0 -> 577 bytes .../autopsy/report/{ => images}/search.png | Bin .../autopsy/report/{ => images}/summary.png | Bin .../report/{ => images}/userbookmarks.png | Bin 59 files changed, 4639 insertions(+), 4199 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/report/ArtifactSelectionDialog.form create mode 100644 Core/src/org/sleuthkit/autopsy/report/ArtifactSelectionDialog.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/BrowserControl.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/DefaultReportConfigurationPanel.form create mode 100644 Core/src/org/sleuthkit/autopsy/report/DefaultReportConfigurationPanel.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/GeneralReportModule.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/Report.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportAction.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportConfiguration.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportExcel.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportFilter.form delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportFilter.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportFilterAction.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportGen.java rename Core/src/org/sleuthkit/autopsy/report/{ReportPanel.form => ReportGenerationPanel.form} (55%) create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportGenerationPanel.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportGenerator.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportModuleException.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportPanel.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportPanelAction.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportProgressPanel.form create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportProgressPanel.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportUtils.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel1.form create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel1.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.form create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportWizardAction.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportWizardIterator.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportWizardPanel1.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportWizardPanel2.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportXLS.java delete mode 100644 Core/src/org/sleuthkit/autopsy/report/ReportXML.java create mode 100644 Core/src/org/sleuthkit/autopsy/report/TableReportModule.java rename Core/src/org/sleuthkit/autopsy/report/{ => images}/bookmarks.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/btn_icon_generate_report.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/cookies.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/devices.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/downloads.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/exif.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/favicon.ico (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/hash.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/history.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/installed.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/keywords.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/logo.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/recent.png (100%) create mode 100644 Core/src/org/sleuthkit/autopsy/report/images/report_cancel.png create mode 100644 Core/src/org/sleuthkit/autopsy/report/images/report_cancel_hover.png create mode 100644 Core/src/org/sleuthkit/autopsy/report/images/report_complete.png create mode 100644 Core/src/org/sleuthkit/autopsy/report/images/report_loading.png rename Core/src/org/sleuthkit/autopsy/report/{ => images}/search.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/summary.png (100%) rename Core/src/org/sleuthkit/autopsy/report/{ => images}/userbookmarks.png (100%) diff --git a/Core/src/org/sleuthkit/autopsy/core/layer.xml b/Core/src/org/sleuthkit/autopsy/core/layer.xml index b41929e55e..8d516e119d 100644 --- a/Core/src/org/sleuthkit/autopsy/core/layer.xml +++ b/Core/src/org/sleuthkit/autopsy/core/layer.xml @@ -198,11 +198,11 @@ - - + + - + @@ -290,24 +290,19 @@ - + - - - + + + - - - - - - + - + -Each reporting submodule implements org.sleuthkit.autopsy.report.ReportModule interface and registers itself in layer.xml +Each reporting submodule implements either the org.sleuthkit.autopsy.report.TableReportModule interface or the org.sleuthkit.autopsy.report.GeneralReportModule interface, and registers itself in layer.xml -Reporting submodule typically interacts with 3 components: -- org.sleuthkit.autopsy.report.ReportConfiguration - to read current reporting configuration set by the user, -- Blackboard API in org.sleuthkit.datamodel.SleuthkitCase class - to traverse and read blackboard artifacts and attributes, -- an API (possibly external/thirdparty API) to convert blackboard artifacts data structures to the desired reporting format. +Implementing either of those interfaces will require the reporting module to implement a number of abstract methods. And depending on the type of report module, different methods will be invoked by the application. + +Table report modules require their sub-classes to override methods to start and end tables, and add rows to those tables. These methods are provided data, generated from a default configuration panel, for the module to report on. Because of this, when creating a table report module one only needs to focus on how to display the data, not how to find it. + +On the other hand, general report modules have a single method to generate the report. This method gives the module freedom to find and process any data it so chooses. General modules also have the ability to provide a configuration panel, allowing the user to choose from various displayed settings. The report module may then use the user's selection to generate a more specific report. + +General modules are also given the responsibility of updating their report's progress bar and processing label in the UI. A progress panel is given to every general report module. It contains basic API to start, stop, and add to the progress bar, as well as update the processing label. The module is also expeted to check the progress bar's status occasionally to see if the user has manually canceled the report. + +\section report_create_module Creating a Report Module +To create a table report module, start off by creating a class and implementing either the TableReportModule interface or the GeneralReportModule interface. + +\subsection report_create_module_table Table Report Modules +If you implement TableReportModule, you should override the methods: +- org.sleuthkit.autopsy.report.TableReportModule::startReport(String path) +- org.sleuthkit.autopsy.report.TableReportModule::endReport() +- org.sleuthkit.autopsy.report.TableReportModule::startDataType(String title) +- org.sleuthkit.autopsy.report.TableReportModule::endDataType() +- org.sleuthkit.autopsy.report.TableReportModule::startSet(String setName) +- org.sleuthkit.autopsy.report.TableReportModule::endSet() +- org.sleuthkit.autopsy.report.TableReportModule::addSetIndex(List sets) +- org.sleuthkit.autopsy.report.TableReportModule::addSetElement(String elementName) +- org.sleuthkit.autopsy.report.TableReportModule::startTable(List titles) +- org.sleuthkit.autopsy.report.TableReportModule::endTable() +- org.sleuthkit.autopsy.report.TableReportModule::addRow(List row) +- org.sleuthkit.autopsy.report.TableReportModule::dateToString(long date) + +When generating table module reports, Autopsy will iterate through a list of user selected data, and call methods such as addRow(List row) for every "row" of data it finds, or startTable(List titles) for every new category it finds. Developers are guarenteed that every start of a data type, set, or table will be followed by an approptiate end. The focus for a table report module should be to take the given information and display it in a user friendly format. See org.sleuthkit.autopsy.report.ReportExcel for an example. + +\subsection report_create_module_general General Report Modules +If you implement GeneralReportModule, the overriden methods will be: +- org.sleuthkit.autopsy.report.GeneralReportModule::generateReport(String reportPath, ReportProgressPanel progressPanel) +- org.sleuthkit.autopsy.report.GeneralReportModule::getConfigurationPanel() + +For general report modules, Autopsy will simply call the generateReport(String reportPath, ReportProgressPanel progressPanel) method and leave it up to the module to aquire and report data in its desired format. The only requirements are that the module saves to the given report path and updates the org.sleuthkit.autopsy.report.ReportProgressPanel as the report progresses. + +When updating the progress panel, it is recommened to update it as infrequently as possible, while still keeping the user informed. If your report processes 100,000 files and you chose to update the UI each time a file is reviewed, the UI would freeze when trying to process all your requests. This would cause problems to not only your reporting module, but to other modules running in parellel. A safer approach would be to update the UI every 1,000 files, or when a certain "category" of the files being processed has changed. For example, the HTML report module increments the progress bar and changes the processing label every time a new Blackboard Artifact Type is being processed. + +Autopsy will also display the panel returned by getConfigurationPanel() in the generate report wizard, when that particular report module is selected. If null is returned, a blank panel will be displayed instead. This panel can be used to allow the user custom controls over the report. + +Typically a general report module should interact with both the Blackboard API in the org.sleuthkit.datamodel.SleuthkitCase class, in addition to an API (possibly external/thirdparty) to convert Blackboard Artifacts to the desired reporting format. + +\subsection report_create_module_layer Registering the Report in layer.xml +Lastly, it is important to register each report module, regardless of the type, to a layer.xml file. This file serves as a globally excessible instance of the report module, and allows all report modules to be recognized abstractly without knowing each class. Without this file, Autopsy will be unable to see your report module. + +An example entry into layer.xml is shown below: +\code + + + + + + + +\endcode + +In the above example, "org-sleuthkit-autopsy-report-ReportHTML" should be replaced with the package based path to your report module. + +It is also important to remember to include a getDefault() method in your report module. As shown in the code above, the instance to each report module is accessed via it's getDefault() method. + +\code +// Static instance of this report +private static MyReport instance; + +// Get the default instance of this report +public static synchronized MyReport getDefault() { + if (instance == null) { + instance = new MyReport(); + } + return instance; +} +\endcode + +Above is an example implementation of the getDefault() method. */ From 5610bfe8609fac1c82ed1af72e76cf2f66a451c3 Mon Sep 17 00:00:00 2001 From: Devin148 Date: Fri, 21 Dec 2012 13:58:22 -0500 Subject: [PATCH 6/6] Update NEWS --- NEWS.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.txt b/NEWS.txt index 80438d494a..5c35395456 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,8 +1,11 @@ ---------------- VERSION Current (dev) -------------- New features: +- Documented report module API Improvements: +- Remake of reporting UI and functionality +- Significant increase in reporting speed Bugfixes: