diff --git a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties index 3636fce22f..6c3e5ce690 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties @@ -52,3 +52,4 @@ CTL_OpenLogFolder=Open Log Folder ShowIngestProgressSnapshotAction.actionName.text=Get Ingest Progress Snapshot OpenPythonModulesFolderAction.actionName.text=Python Plugins OpenPythonModulesFolderAction.errorMsg.folderNotFound=Python plugins folder not found: {0} +CTL_OpenPythonModulesFolderAction=Python Plugins \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties index edf52bb547..480522e021 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties @@ -45,4 +45,7 @@ GetTagNameDialog.dupTagErr=\u91CD\u8907\u30BF\u30B0\u306E\u30A8\u30E9\u30FC AddContentTagAction.cannotApplyTagErr=\u30BF\u30B0\u3092\u9069\u7528\u3067\u304D\u307E\u305B\u3093 OpenLogFolder.error1=\u30ED\u30B0\u30D5\u30A1\u30A4\u30EB\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F\uFF1A{0} CTL_OpenLogFolder=\u30ED\u30B0\u30D5\u30A9\u30EB\u30C0\u3092\u958B\u304F -ShowIngestProgressSnapshotAction.actionName.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30D7\u30ED\u30B0\u30EC\u30B9\u306E\u30B9\u30CA\u30C3\u30D7\u30B7\u30E7\u30C3\u30C8\u3092\u53D6\u5F97 \ No newline at end of file +ShowIngestProgressSnapshotAction.actionName.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30D7\u30ED\u30B0\u30EC\u30B9\u306E\u30B9\u30CA\u30C3\u30D7\u30B7\u30E7\u30C3\u30C8\u3092\u53D6\u5F97 +CTL_OpenPythonModulesFolderAction=Python\u30D7\u30E9\u30B0\u30A4\u30F3 +OpenPythonModulesFolderAction.actionName.text=Python\u30D7\u30E9\u30B0\u30A4\u30F3 +OpenPythonModulesFolderAction.errorMsg.folderNotFound=Python\u30D7\u30E9\u30B0\u30A4\u30F3\u30D5\u30A9\u30EB\u30C0\u30FC\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F\uFF1A{0} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java b/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java index 68195c299a..16a3ea1891 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java @@ -38,7 +38,6 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; @ActionID(category = "Tools", id = "org.sleuthkit.autopsy.actions.OpenPythonModulesFolderAction") @ActionRegistration(displayName = "#CTL_OpenPythonModulesFolderAction", lazy = false) @ActionReference(path = "Menu/Tools", position = 1400) -@NbBundle.Messages("CTL_OpenPythonModulesFolderAction=Python Plugins") public final class OpenPythonModulesFolderAction extends SystemAction implements ActionListener { private static final Logger logger = Logger.getLogger(OpenPythonModulesFolderAction.class.getName()); diff --git a/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java b/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java index fcedf212b2..9a9e1995e3 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java @@ -46,7 +46,6 @@ public final class ShowIngestProgressSnapshotAction extends SystemAction impleme @Override public void actionPerformed(ActionEvent e) { IngestProgressSnapshotDialog dialog = new IngestProgressSnapshotDialog(); - dialog.setVisible(true); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 268404faa0..fe70d57eeb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,10 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel> props = new ArrayList<>(propertiesAcc); + + /* OutlineView makes the first column be the result of node.getDisplayName with the icon. This + * duplicates our first column, which is the file name, etc. So, pop that property off the list, but + * use its display name as the header for the column so that the header can change depending on the + * type of data being displayed. + * + * NOTE: This assumes that the first property is always the one tha duplicates getDisplayName(). This + * seems like a big assumption and could be made more robust. + */ if (props.size() > 0) { Node.Property prop = props.remove(0); ((DefaultOutlineModel) ov.getOutline().getOutlineModel()).setNodesColumnLabel(prop.getDisplayName()); } - // *********** Make the TreeTableView to be sortable *************** - - //First property column is sortable, but also sorted initially, so - //initially this one will have the arrow icon: - if (props.size() > 0) { - props.get(0).setValue("TreeColumnTTV", Boolean.TRUE); // Identifies special property representing first (tree) column. NON-NLS - props.get(0).setValue("SortingColumnTTV", Boolean.TRUE); // TreeTableView should be initially sorted by this property column. NON-NLS - } - - // The rest of the columns are sortable, but not initially sorted, - // so initially will have no arrow icon: + // Get the columns setup with respect to names and sortability String[] propStrings = new String[props.size() * 2]; for (int i = 0; i < props.size(); i++) { props.get(i).setValue("ComparableColumnTTV", Boolean.TRUE); //NON-NLS + //First property column is sorted initially + if (i == 0) { + props.get(i).setValue("TreeColumnTTV", Boolean.TRUE); // Identifies special property representing first (tree) column. NON-NLS + props.get(i).setValue("SortingColumnTTV", Boolean.TRUE); // TreeTableView should be initially sorted by this property column. NON-NLS + } propStrings[2 * i] = props.get(i).getName(); propStrings[2 * i + 1] = props.get(i).getDisplayName(); } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java index a5c01c9d42..da0423735b 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java @@ -30,6 +30,8 @@ import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryUsage; import java.nio.charset.Charset; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -211,37 +213,31 @@ public class PlatformUtil { * * @param resourceClass class in the same package as the resourceFile to * extract - * @param resourceFile resource file name to extract + * @param resourceFileName Name of the resource file to extract * @param overWrite true to overwrite an existing resource * @return true if extracted, false otherwise (if file already exists) * @throws IOException exception thrown if extract the file failed for IO * reasons */ - public static boolean extractResourceToUserConfigDir(final Class resourceClass, final String resourceFile, boolean overWrite) throws IOException { - final File userDir = new File(getUserConfigDirectory()); - - final File resourceFileF = new File(userDir + File.separator + resourceFile); - if (resourceFileF.exists() && !overWrite) { + public static boolean extractResourceToUserConfigDir(final Class resourceClass, final String resourceFileName, boolean overWrite) throws IOException { + Path resourceFilePath = Paths.get(getUserConfigDirectory(), resourceFileName); + final File resourceFile = resourceFilePath.toFile(); + if (resourceFile.exists() && !overWrite) { return false; } - resourceFileF.getParentFile().mkdirs(); + InputStream inputStream = resourceClass.getResourceAsStream(resourceFileName); + if (null == inputStream) { + return false; + } - InputStream inputStream = resourceClass.getResourceAsStream(resourceFile); - - OutputStream out = null; + resourceFile.getParentFile().mkdirs(); try (InputStream in = new BufferedInputStream(inputStream)) { - - OutputStream outFile = new FileOutputStream(resourceFileF); - out = new BufferedOutputStream(outFile); - int readBytes; - while ((readBytes = in.read()) != -1) { - out.write(readBytes); - } - } finally { - if (out != null) { - out.flush(); - out.close(); + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(resourceFile))) { + int readBytes; + while ((readBytes = in.read()) != -1) { + out.write(readBytes); + } } } return true; diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java index a997f01dd5..cc055638d7 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java @@ -25,6 +25,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; +import java.nio.file.Paths; import java.util.logging.Level; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; @@ -54,6 +55,101 @@ import org.xml.sax.SAXException; */ public class XMLUtil { + /** + * Creates a W3C DOM. + * + * @return The document object. + * @throws ParserConfigurationException + */ + public static Document createDocument() throws ParserConfigurationException { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + return builder.newDocument(); + } + + /** + * Loads an XML document into a WC3 DOM and validates it using a schema + * packaged as a class resource. + * + * @param The name of the class associated with the resource. + * @param docPath The full path to the XML document. + * @param clazz The class associated with the schema resource. + * @param schemaResourceName The name of the schema resource. + * @return The WC3 DOM document object. + * @throws IOException + * @throws ParserConfigurationException + * @throws SAXException + */ + public static Document loadDocument(String docPath, Class clazz, String schemaResourceName) throws IOException, ParserConfigurationException, SAXException { + Document doc = loadDocument(docPath); + validateDocument(doc, clazz, schemaResourceName); + return doc; + } + + /** + * Loads an XML document into a WC3 DOM. + * + * @param docPath The full path to the XML document. + * @return The WC3 DOM document object. + * @throws ParserConfigurationException + * @throws SAXException + * @throws IOException + */ + public static Document loadDocument(String docPath) throws ParserConfigurationException, SAXException, IOException { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document doc = builder.parse(new FileInputStream(docPath)); + return doc; + } + + /** + * Validates a WC3 DOM using a schema packaged as a class resource. + * + * @param doc + * @param clazz + * @param schemaResourceName + * @throws SAXException + * @throws IOException + */ + public static void validateDocument(final Document doc, Class clazz, String schemaResourceName) throws SAXException, IOException { + PlatformUtil.extractResourceToUserConfigDir(clazz, schemaResourceName, false); + File schemaFile = new File(Paths.get(PlatformUtil.getUserConfigDirectory(), schemaResourceName).toAbsolutePath().toString()); + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = schemaFactory.newSchema(schemaFile); + Validator validator = schema.newValidator(); + validator.validate(new DOMSource(doc), new DOMResult()); + } + + /** + * Saves a WC3 DOM by writing it to an XML document. + * + * @param doc The WC3 DOM document object. + * @param docPath The full path to the XML document. + * @param encoding Encoding scheme to use for the XML document, e.g., + * "UTF-8." + * @throws TransformerConfigurationException + * @throws FileNotFoundException + * @throws UnsupportedEncodingException + * @throws TransformerException + * @throws IOException + */ + public static void saveDocument(final Document doc, String encoding, String docPath) throws TransformerConfigurationException, FileNotFoundException, UnsupportedEncodingException, TransformerException, IOException { + TransformerFactory xf = TransformerFactory.newInstance(); + xf.setAttribute("indent-number", 1); //NON-NLS + Transformer xformer = xf.newTransformer(); + xformer.setOutputProperty(OutputKeys.METHOD, "xml"); //NON-NLS + xformer.setOutputProperty(OutputKeys.INDENT, "yes"); //NON-NLS + xformer.setOutputProperty(OutputKeys.ENCODING, encoding); + xformer.setOutputProperty(OutputKeys.STANDALONE, "yes"); //NON-NLS + xformer.setOutputProperty(OutputKeys.VERSION, "1.0"); + File file = new File(docPath); + try (FileOutputStream stream = new FileOutputStream(file)) { + Result out = new StreamResult(new OutputStreamWriter(stream, encoding)); + xformer.transform(new DOMSource(doc), out); + stream.flush(); + } + } + /** * Utility to validate XML files against pre-defined schema files. * @@ -71,6 +167,7 @@ public class XMLUtil { * IngestModuleLoader. * */ + // TODO: Deprecate. public static boolean xmlIsValid(DOMSource xmlfile, Class clazz, String schemaFile) { try { PlatformUtil.extractResourceToUserConfigDir(clazz, schemaFile, false); @@ -108,6 +205,7 @@ public class XMLUtil { * IngestModuleLoader. * */ + // TODO: Deprecate. public static boolean xmlIsValid(Document doc, Class clazz, String type) { DOMSource dms = new DOMSource(doc); return xmlIsValid(dms, clazz, type); @@ -119,6 +217,7 @@ public class XMLUtil { * @param clazz the class this method is invoked from * @param xmlPath the full path to the file to load */ + // TODO: Deprecate. public static Document loadDoc(Class clazz, String xmlPath) { DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); Document ret = null; @@ -143,6 +242,7 @@ public class XMLUtil { * @param xmlPath the full path to the file to load * @param xsdPath the full path to the file to validate against */ + // TODO: Deprecate public static Document loadDoc(Class clazz, String xmlPath, String xsdPath) { Document ret = loadDoc(clazz, xmlPath); if (!XMLUtil.xmlIsValid(ret, clazz, xsdPath)) { @@ -159,9 +259,10 @@ public class XMLUtil { * @param encoding to encoding, such as "UTF-8", to encode the file with * @param doc the document to save */ + // TODO: Deprecate. public static boolean saveDoc(Class clazz, String xmlPath, String encoding, final Document doc) { TransformerFactory xf = TransformerFactory.newInstance(); - xf.setAttribute("indent-number", new Integer(1)); //NON-NLS + xf.setAttribute("indent-number", 1); //NON-NLS boolean success = false; try { Transformer xformer = xf.newTransformer(); @@ -191,4 +292,5 @@ public class XMLUtil { } return success; } + } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java index babc47cde8..3cc523b4dc 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java @@ -88,7 +88,7 @@ public class ImageNode extends AbstractContentNode { ss.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ImageNode.createSheet.name.name"), NbBundle.getMessage(this.getClass(), "ImageNode.createSheet.name.displayName"), NbBundle.getMessage(this.getClass(), "ImageNode.createSheet.name.desc"), - getName())); + getDisplayName())); return s; } diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties index eae8c24793..90ad7a3ffe 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties @@ -1,3 +1,4 @@ +CTL_PerformancePanelAction=Performance Diagnostics PerformancePanel.jLabel1.text=Image Reading: PerformancePanel.jLabel2.text=Database Reading: PerformancePanel.jLabel3.text=CPU: @@ -8,4 +9,21 @@ PerformancePanel.startButton.text=Start PerformancePanel.statusLabel.text=\ PerformancePanel.jLabel4.text=File Reading: PerformancePanel.fileReadLabel.text=\ -PerformancePanel.jLabel5.text=This panel performs a series of tests to help identify bottlenecks in the system. +PerformancePanel.jLabel5.text=This panel performs a series of tests to help identify bottlenecks in the system. +PerformancePanel.title=Performance Diagnostics +PerformancePanel.cpuTest.basemsg=Running CPU Test +PerformancePanel.cpuTest.cpuLabel.md5AlgNotFound.text=MD5 Algorithm not found +PerformancePanel.cpuTest.cpuLabel.MBHashedPerSec.text={0} MB hashed / sec +PerformancePanel.imgTest.statusMsg.runningImgReadTest.text=Running Image Reading Test +PerformancePanel.label.caseNotOpen.text=Case Not Open +PerformancePanel.label.noImgInCase.text=No Images In Case +PerformancePanel.ImgTest.imgLabel.MBReadPerSec.text={0} MB read / sec ({1}) +PerformancePanel.FileReadTest.fileReadLabel.skipped.text=Skipped +PerformancePanel.FileReadTest.statusMsg.runningFileReadTest.text=Running File Reading Test +PerformancePanel.fileReadLabel.imgPathNotExist.text=Image Path Doesn't Exist +PerformancePanel.fileReadLabel.errMakeFileReader.text=Error making file reader +PerformancePanel.ImgTest.fileReadLabel.MBReadPerSec.text={0} MB read / sec ({1}) +PerformancePanel.dbTest.status.running=Running DB Test +PerformancePanel.dbTest.dbLabel.recordsPerSec.text={0} records / sec +PerformancePanel.dbTest.dbLabel.errPerformQuery.text=Error Performing Query +PerformancePanel.done.statusMsg.err.text=Error\: {0} diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties new file mode 100644 index 0000000000..09ec101d1d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties @@ -0,0 +1,22 @@ +CTL_PerformancePanelAction=\u30D1\u30D5\u30A9\u30FC\u30DE\u30F3\u30B9\u8A3A\u65AD +PerformancePanel.cpuTest.basemsg=CPU\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.cpuTest.cpuLabel.MBHashedPerSec.text=\uFF11\u79D2\u3054\u3068\u306B{0}MB\u30CF\u30C3\u30B7\u30E5\u5B8C\u4E86 +PerformancePanel.cpuTest.cpuLabel.md5AlgNotFound.text=MD5\u30A2\u30EB\u30B4\u30EA\u30BA\u30E0\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F +PerformancePanel.dbTest.dbLabel.errPerformQuery.text=\u30AF\u30A8\u30EA\u5B9F\u884C\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F +PerformancePanel.dbTest.dbLabel.recordsPerSec.text=\uFF11\u79D2\u3054\u3068\u306B{0}\u30EC\u30B3\u30FC\u30C9 +PerformancePanel.dbTest.status.running=DB\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.done.statusMsg.err.text=\u30A8\u30E9\u30FC\uFF1A{0} +PerformancePanel.fileReadLabel.errMakeFileReader.text=\u30D5\u30A1\u30A4\u30EB\u30EA\u30FC\u30C0\u30FC\u4F5C\u6210\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F +PerformancePanel.fileReadLabel.imgPathNotExist.text=\u30A4\u30E1\u30FC\u30B8\u30D1\u30B9\u304C\u5B58\u5728\u3057\u307E\u305B\u3093 +PerformancePanel.FileReadTest.fileReadLabel.skipped.text=\u30B9\u30AD\u30C3\u30D7\u3055\u308C\u307E\u3057\u305F +PerformancePanel.FileReadTest.statusMsg.runningFileReadTest.text=\u30D5\u30A1\u30A4\u30EB\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.imgTest.statusMsg.runningImgReadTest.text=\u30A4\u30E1\u30FC\u30B8\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.jLabel1.text=\u30A4\u30E1\u30FC\u30B8\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\uFF1A +PerformancePanel.jLabel2.text=\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\uFF1A +PerformancePanel.jLabel3.text=CPU\uFF1A +PerformancePanel.jLabel4.text=\u30D5\u30A1\u30A4\u30EB\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\uFF1A +PerformancePanel.jLabel5.text=\u30B7\u30B9\u30C6\u30E0\u306E\u30DC\u30C8\u30EB\u30CD\u30C3\u30AF\u3092\u7279\u5B9A\u3059\u308B\u4E00\u9023\u306E\u30C6\u30B9\u30C8\u3092\u884C\u3046\u30D1\u30CD\u30EB\u3067\u3059\u3002 +PerformancePanel.label.caseNotOpen.text=\u30B1\u30FC\u30B9\u304C\u958B\u3044\u3066\u3044\u307E\u305B\u3093 +PerformancePanel.label.noImgInCase.text=\u30B1\u30FC\u30B9\u306B\u30A4\u30E1\u30FC\u30B8\u304C\u3042\u308A\u307E\u305B\u3093 +PerformancePanel.startButton.text=\u958B\u59CB +PerformancePanel.title=\u30D1\u30D5\u30A9\u30FC\u30DE\u30F3\u30B9\u8A3A\u65AD \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java index e2cc873095..ace829ee2c 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java @@ -34,6 +34,8 @@ import java.util.concurrent.ExecutionException; import javax.swing.JFrame; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; + +import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.datamodel.AbstractFile; @@ -48,7 +50,8 @@ public class PerformancePanel extends javax.swing.JDialog { * Creates new form PerformancePanel */ public PerformancePanel() { - super((JFrame) WindowManager.getDefault().getMainWindow(), "Performance Diagnostics", true); + super((JFrame) WindowManager.getDefault().getMainWindow(), + NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.title"), true); initComponents(); } @@ -258,14 +261,15 @@ public class PerformancePanel extends javax.swing.JDialog { private void doCpuTest() { - final String msg = "Running CPU Test"; + final String msg = NbBundle.getMessage(this.getClass(), "PerformancePanel.cpuTest.basemsg"); MessageDigest md; long start = new Date().getTime(); try { - md = MessageDigest.getInstance("MD5"); + md = MessageDigest.getInstance("MD5"); // NON-NLS } catch (NoSuchAlgorithmException ex) { - setCpuLabel("MD5 Algo not found"); + setCpuLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.cpuTest.cpuLabel.md5AlgNotFound.text")); return; } @@ -285,20 +289,22 @@ public class PerformancePanel extends javax.swing.JDialog { long end = new Date().getTime(); cpuStats = (bytesRead / (1024 * 1024)) / ((end - start) / 1000); - setCpuLabel(cpuStats + " MB hashed / sec"); + setCpuLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.cpuTest.cpuLabel.MBHashedPerSec.text", + cpuStats)); setStatusMsg(""); } private void doImgTest() { imgReadStats = 0; - setStatusMsg("Running Image Reading Test"); + setStatusMsg( + NbBundle.getMessage(this.getClass(), "PerformancePanel.imgTest.statusMsg.runningImgReadTest.text")); Case curCase; try { curCase = Case.getCurrentCase(); } catch (Exception e) { - setImgLabel("Case Not Open"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.caseNotOpen.text")); setStatusMsg(""); return; } @@ -307,7 +313,7 @@ public class PerformancePanel extends javax.swing.JDialog { try { dataSources = curCase.getDataSources(); } catch (TskCoreException ex) { - setImgLabel("No Images In Case"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } @@ -318,7 +324,7 @@ public class PerformancePanel extends javax.swing.JDialog { } } if (image == null) { - setImgLabel("No Images In Case"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } @@ -356,25 +362,30 @@ public class PerformancePanel extends javax.swing.JDialog { imgReadStats = (bytesRead / (1024 * 1024)) / elapsed; else imgReadStats = 0; - setImgLabel(imgReadStats + " MB read / sec (" + bytesRead + ")"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.ImgTest.imgLabel.MBReadPerSec.text", + imgReadStats, bytesRead)); setStatusMsg(""); } private void doFileReadTest() { fileReadStats = 0; - + + // TODO: this is always true. Why display a "Skipped" label and then go on to run the test? if (true) { - setFileReadLabel("Skipped"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.FileReadTest.fileReadLabel.skipped.text")); } - setStatusMsg("Running File Reading Test"); + setStatusMsg(NbBundle.getMessage(this.getClass(), + "PerformancePanel.FileReadTest.statusMsg.runningFileReadTest.text")); Case curCase; try { curCase = Case.getCurrentCase(); } catch (Exception e) { - setFileReadLabel("Case Not Open"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.label.caseNotOpen.text")); setStatusMsg(""); return; } @@ -383,7 +394,8 @@ public class PerformancePanel extends javax.swing.JDialog { try { dataSources = curCase.getDataSources(); } catch (TskCoreException ex) { - setFileReadLabel("No Images In Case"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } @@ -394,14 +406,16 @@ public class PerformancePanel extends javax.swing.JDialog { } } if (image == null) { - setFileReadLabel("No Images In Case"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } File file = new File(image.getPaths()[0]); if (file.exists() == false) { - setFileReadLabel("Image Path Doesn't Exist"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.fileReadLabel.imgPathNotExist.text")); setStatusMsg(""); return; } @@ -410,7 +424,8 @@ public class PerformancePanel extends javax.swing.JDialog { try { fileReader = new FileReader(file); } catch (FileNotFoundException ex) { - setFileReadLabel("Error making file reader"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.fileReadLabel.errMakeFileReader.text")); setStatusMsg(""); return; } @@ -451,20 +466,22 @@ public class PerformancePanel extends javax.swing.JDialog { fileReadStats = (bytesRead / (1024 * 1024)) / elapsed; else fileReadStats = 0; - setFileReadLabel(fileReadStats + " MB read / sec (" + bytesRead + ")"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.ImgTest.fileReadLabel.MBReadPerSec.text", + fileReadStats, bytesRead)); setStatusMsg(""); } private void doDbTest() { dbStats = 0; - setStatusMsg("Running DB Test"); + setStatusMsg(NbBundle.getMessage(this.getClass(), "PerformancePanel.dbTest.status.running")); Case curCase; try { curCase = Case.getCurrentCase(); } catch (Exception e) { - setDbLabel("Case Not Open"); + setDbLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.caseNotOpen.text")); return; } @@ -472,7 +489,7 @@ public class PerformancePanel extends javax.swing.JDialog { SleuthkitCase tskCase = curCase.getSleuthkitCase(); long start = new Date().getTime(); - List files = tskCase.findAllFilesWhere("obj_id < 50000"); + List files = tskCase.findAllFilesWhere("obj_id < 50000"); // NON-NLS long end = new Date().getTime(); long elapsed = (end - start) / 1000; @@ -481,9 +498,10 @@ public class PerformancePanel extends javax.swing.JDialog { else dbStats = 0; - setDbLabel(dbStats + " records / sec"); + setDbLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.dbTest.dbLabel.recordsPerSec.text", + dbStats)); } catch (TskCoreException ex) { - setDbLabel("Error Performing Query"); + setDbLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.dbTest.dbLabel.errPerformQuery.text")); } setStatusMsg(""); @@ -494,7 +512,8 @@ public class PerformancePanel extends javax.swing.JDialog { try { get(); } catch (InterruptedException | ExecutionException ex) { - setStatusMsg("Error: " + ex.getMessage()); + setStatusMsg(NbBundle.getMessage(this.getClass(), "PerformancePanel.done.statusMsg.err.text", + ex.getMessage())); } startButton.setEnabled(true); } diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java index 0f962e6abf..54559d0f9c 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java @@ -35,7 +35,6 @@ import org.openide.util.NbBundle.Messages; displayName = "#CTL_PerformancePanelAction" ) @ActionReference(path = "Menu/Help", position = 1437) -@Messages("CTL_PerformancePanelAction=Performance Diagnostics") public final class PerformancePanelAction implements ActionListener { @Override diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java index 72681fba2a..fd2fce803a 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java @@ -91,17 +91,28 @@ public final class ExtractAction extends AbstractAction { } } - private void extractFile(ActionEvent e, AbstractFile source) { + /** + * Called when user has selected a single file to extract + * @param e + * @param selectedFile Selected file + */ + private void extractFile(ActionEvent e, AbstractFile selectedFile) { JFileChooser fileChooser = new JFileChooser(); fileChooser.setCurrentDirectory(new File(Case.getCurrentCase().getExportDirectory())); - fileChooser.setSelectedFile(new File(source.getName())); + // If there is an attribute name, change the ":". Otherwise the extracted file will be hidden + fileChooser.setSelectedFile(new File(selectedFile.getName().replace(':', '_'))); if (fileChooser.showSaveDialog((Component)e.getSource()) == JFileChooser.APPROVE_OPTION) { ArrayList fileExtractionTasks = new ArrayList<>(); - fileExtractionTasks.add(new FileExtractionTask(source, fileChooser.getSelectedFile())); - doFileExtraction(e, fileExtractionTasks); + fileExtractionTasks.add(new FileExtractionTask(selectedFile, fileChooser.getSelectedFile())); + runExtractionTasks(e, fileExtractionTasks); } } + /** + * Called when a user has selected multiple files to extract + * @param e + * @param selectedFiles Selected files + */ private void extractFiles(ActionEvent e, Collection selectedFiles) { JFileChooser folderChooser = new JFileChooser(); folderChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); @@ -120,15 +131,17 @@ public final class ExtractAction extends AbstractAction { } } + // make a task for each file ArrayList fileExtractionTasks = new ArrayList<>(); for (AbstractFile source : selectedFiles) { - fileExtractionTasks.add(new FileExtractionTask(source, new File(destinationFolder, source.getId() + "-" + source.getName()))); + // If there is an attribute name, change the ":". Otherwise the extracted file will be hidden + fileExtractionTasks.add(new FileExtractionTask(source, new File(destinationFolder, source.getId() + "-" + source.getName().replace(':', '_')))); } - doFileExtraction(e, fileExtractionTasks); + runExtractionTasks(e, fileExtractionTasks); } } - private void doFileExtraction(ActionEvent e, ArrayList fileExtractionTasks) { + private void runExtractionTasks(ActionEvent e, ArrayList fileExtractionTasks) { // verify all of the sources and destinations are OK for (Iterator it = fileExtractionTasks.iterator(); it.hasNext(); ) { @@ -162,6 +175,7 @@ public final class ExtractAction extends AbstractAction { } } + // launch a thread to do the work if (!fileExtractionTasks.isEmpty()) { try { FileExtracter extracter = new FileExtracter(fileExtractionTasks); @@ -187,6 +201,9 @@ public final class ExtractAction extends AbstractAction { } } + /** + * Thread that does the actual extraction work + */ private class FileExtracter extends SwingWorker { private Logger logger = Logger.getLogger(FileExtracter.class.getName()); private ProgressHandle progress; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 11b5773f20..ab4fd56ddf 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -49,6 +49,7 @@ IngestMessageTopComponent.msgDlg.ingestRpt.text=Ingest Report IngestMonitor.mgrErrMsg.lowDiskSpace.title=Ingest stopped - low disk space on {0} IngestMonitor.mgrErrMsg.lowDiskSpace.msg=Stopping ingest due to low disk space on disk {0}. \nEnsure the Case drive has at least 1GB free space and restart ingest. IngestManager.StartIngestJobsTask.run.displayName=Starting ingest +IngestManager.StartIngestJobsTask.run.progressingDisplayName=Starting ingest of {0} IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...) IngestMessagePanel.sortByComboBox.model.time=Time IngestMessagePanel.sortByComboBox.model.priority=Priority diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java new file mode 100644 index 0000000000..a1408ef6c2 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -0,0 +1,1123 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import javax.swing.JOptionPane; +import org.netbeans.api.progress.ProgressHandle; +import org.netbeans.api.progress.ProgressHandleFactory; +import org.openide.util.Cancellable; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; + +/** + * Encapsulates a data source and the ingest module pipelines used to process + * it. + */ +final class DataSourceIngestJob { + + private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName()); + + /** + * These fields define a data source ingest job: the parent ingest job, an + * ID, the user's ingest job settings, and the data source to be processed. + */ + private final IngestJob parentJob; + private static final AtomicLong nextJobId = new AtomicLong(0L); + private final long id; + private final IngestJobSettings settings; + private final Content dataSource; + + /** + * A data source ingest job runs in stages. + */ + private static enum Stages { + + /** + * Setting up for processing. + */ + INITIALIZATION, + /** + * Running high priority data source level ingest modules and file level + * ingest modules. + */ + FIRST, + /** + * Running lower priority, usually long-running, data source level + * ingest modules. + */ + SECOND, + /** + * Cleaning up. + */ + FINALIZATION + }; + private Stages stage = DataSourceIngestJob.Stages.INITIALIZATION; + private final Object stageCompletionCheckLock = new Object(); + + /** + * A data source ingest job has separate data source level ingest module + * pipelines for the first and second processing stages. Longer running, + * lower priority modules belong in the second stage pipeline, although this + * cannot be enforced. Note that the pipelines for both stages are created + * at job start up to allow for verification that they both can be started + * up without errors. + */ + private final Object dataSourceIngestPipelineLock = new Object(); + private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; + private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; + private DataSourceIngestPipeline currentDataSourceIngestPipeline; + + /** + * A data source ingest job has a collection of identical file level ingest + * module pipelines, one for each file level ingest thread in the ingest + * manager. A blocking queue is used to dole out the pipelines to the + * threads and an ordinary list is used when the ingest job needs to access + * the pipelines to query their status. + */ + private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); + private final List fileIngestPipelines = new ArrayList<>(); + + /** + * A data source ingest job supports cancellation of either the currently + * running data source level ingest module or the entire ingest job. + * + * TODO: The currentDataSourceIngestModuleCancelled field and all of the + * code concerned with it is a hack to avoid an API change. The next time an + * API change is legal, a cancel() method needs to be added to the + * IngestModule interface and this field should be removed. The "ingest job + * is canceled" queries should also be removed from the IngestJobContext + * class. + */ + private volatile boolean currentDataSourceIngestModuleCancelled; + private volatile boolean cancelled; + + /** + * A data source ingest job uses the task scheduler singleton to create and + * queue the ingest tasks that make up the job. + */ + private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + + /** + * A data source ingest job can run interactively using NetBeans progress + * handles. + */ + private final boolean runInteractively; + + /** + * A data source ingest job uses these fields to report data source level + * ingest progress. + */ + private final Object dataSourceIngestProgressLock = new Object(); + private ProgressHandle dataSourceIngestProgress; + + /** + * A data source ingest job uses these fields to report file level ingest + * progress. + */ + private final Object fileIngestProgressLock = new Object(); + private final List filesInProgress = new ArrayList<>(); + private long estimatedFilesToProcess; + private long processedFiles; + private ProgressHandle fileIngestProgress; + + /** + * A data source ingest job uses this field to report its creation time. + */ + private final long createTime; + + /** + * Constructs an object that encapsulates a data source and the ingest + * module pipelines used to process it. + * + * @param parentJob The ingest job of which this data source ingest job is a + * part. + * @param dataSource The data source to be ingested. + * @param settings The settings for the ingest job. + * @param runInteractively Whether or not this job should use NetBeans + * progress handles. + */ + DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) { + this.parentJob = parentJob; + this.id = DataSourceIngestJob.nextJobId.getAndIncrement(); + this.dataSource = dataSource; + this.settings = settings; + this.runInteractively = runInteractively; + this.createTime = new Date().getTime(); + this.createIngestPipelines(); + } + + /** + * Creates the file and data source ingest pipelines. + */ + private void createIngestPipelines() { + List ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates(); + + /** + * Make mappings of ingest module factory class names to templates. + */ + Map dataSourceModuleTemplates = new HashMap<>(); + Map fileModuleTemplates = new HashMap<>(); + for (IngestModuleTemplate template : ingestModuleTemplates) { + if (template.isDataSourceIngestModuleTemplate()) { + dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + if (template.isFileIngestModuleTemplate()) { + fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + } + + /** + * Use the mappings and the ingest pipelines configuration to create + * ordered lists of ingest module templates for each ingest pipeline. + */ + IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); + List firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + + /** + * Add any module templates that were not specified in the pipelines + * configuration to an appropriate pipeline - either the first stage + * data source ingest pipeline or the file ingest pipeline. + */ + for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { + firstStageDataSourceModuleTemplates.add(template); + } + for (IngestModuleTemplate template : fileModuleTemplates.values()) { + fileIngestModuleTemplates.add(template); + } + + /** + * Construct the data source ingest pipelines. + */ + this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); + this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); + + /** + * Construct the file ingest pipelines, one per file ingest thread. + */ + try { + int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); + this.fileIngestPipelinesQueue.put(pipeline); + this.fileIngestPipelines.add(pipeline); + } + } catch (InterruptedException ex) { + /** + * The current thread was interrupted while blocked on a full queue. + * Blocking should actually never happen here, but reset the + * interrupted flag rather than just swallowing the exception. + */ + Thread.currentThread().interrupt(); + } + } + + /** + * Uses an input collection of ingest module templates and a pipeline + * configuration, i.e., an ordered list of ingest module factory class + * names, to create an ordered output list of ingest module templates for an + * ingest pipeline. The ingest module templates are removed from the input + * collection as they are added to the output collection. + * + * @param ingestModuleTemplates A mapping of ingest module factory class + * names to ingest module templates. + * @param pipelineConfig An ordered list of ingest module factory class + * names representing an ingest pipeline. + * @return An ordered list of ingest module templates, i.e., an + * uninstantiated pipeline. + */ + private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { + List templates = new ArrayList<>(); + for (String moduleClassName : pipelineConfig) { + if (ingestModuleTemplates.containsKey(moduleClassName)) { + templates.add(ingestModuleTemplates.remove(moduleClassName)); + } + } + return templates; + } + + /** + * Gets the identifier of this job. + * + * @return The job identifier. + */ + long getId() { + return this.id; + } + + /** + * Gets the data source to be ingested by this job. + * + * @return A Content object representing the data source. + */ + Content getDataSource() { + return this.dataSource; + } + + /** + * Queries whether or not unallocated space should be processed as part of + * this job. + * + * @return True or false. + */ + boolean shouldProcessUnallocatedSpace() { + return this.settings.getProcessUnallocatedSpace(); + } + + /** + * Checks to see if this job has at least one ingest pipeline. + * + * @return True or false. + */ + boolean hasIngestPipeline() { + return this.hasFirstStageDataSourceIngestPipeline() + || this.hasFileIngestPipeline() + || this.hasSecondStageDataSourceIngestPipeline(); + } + + /** + * Checks to see if this job has a first stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasFirstStageDataSourceIngestPipeline() { + return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if this job has a second stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasSecondStageDataSourceIngestPipeline() { + return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if this job has a file level ingest pipeline. + * + * @return True or false. + */ + private boolean hasFileIngestPipeline() { + if (!this.fileIngestPipelines.isEmpty()) { + return !this.fileIngestPipelines.get(0).isEmpty(); + } + return false; + } + + /** + * Starts up the ingest pipelines for this job. + * + * @return A collection of ingest module startup errors, empty on success. + */ + List start() { + List errors = startUpIngestPipelines(); + if (errors.isEmpty()) { + if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { + this.startFirstStage(); + } else if (this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } + } + return errors; + } + + /** + * Starts up each of the ingest pipelines for this job to collect any file + * and data source level ingest modules errors that might occur. + * + * @return A collection of ingest module startup errors, empty on success. + */ + private List startUpIngestPipelines() { + List errors = new ArrayList<>(); + + // Start up the first stage data source ingest pipeline. + errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); + + // Start up the second stage data source ingest pipeline. + errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); + + // Start up the file ingest pipelines (one per file ingest thread). + for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) { + errors.addAll(pipeline.startUp()); + if (!errors.isEmpty()) { + // If there are start up errors, the ingest job will not proceed + // and the errors will ultimately be reported to the user for + // possible remedy so shut down the pipelines now that an + // attempt has been made to start up the data source ingest + // pipeline and at least one copy of the file ingest pipeline. + // pipeline. There is no need to complete starting up all of the + // file ingest pipeline copies since any additional start up + // errors are likely redundant. + while (!this.fileIngestPipelinesQueue.isEmpty()) { + pipeline = this.fileIngestPipelinesQueue.poll(); + List shutDownErrors = pipeline.shutDown(); + if (!shutDownErrors.isEmpty()) { + logIngestModuleErrors(shutDownErrors); + } + } + break; + } + } + + logIngestModuleErrors(errors); + return errors; + } + + /** + * Starts the first stage of this job. + */ + private void startFirstStage() { + this.stage = DataSourceIngestJob.Stages.FIRST; + + if (this.hasFileIngestPipeline()) { + synchronized (this.fileIngestProgressLock) { + this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + } + } + + if (this.runInteractively) { + /** + * Start one or both of the first stage ingest progress bars. + */ + if (this.hasFirstStageDataSourceIngestPipeline()) { + this.startDataSourceIngestProgressBar(); + } + if (this.hasFileIngestPipeline()) { + this.startFileIngestProgressBar(); + } + } + + /** + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + } + + /** + * Schedule the first stage tasks. + */ + if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); + } else if (this.hasFirstStageDataSourceIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this); + + /** + * No data source ingest task has been scheduled for this stage, and + * it is possible, if unlikely, that no file ingest tasks were + * actually scheduled since there are files that get filtered out by + * the tasks scheduler. In this special case, an ingest thread will + * never get to check for completion of this stage of the job, so do + * it now. + */ + this.checkForStageCompleted(); + } + } + + /** + * Starts the second stage of this ingest job. + */ + private void startSecondStage() { + this.stage = DataSourceIngestJob.Stages.SECOND; + if (this.runInteractively) { + this.startDataSourceIngestProgressBar(); + } + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + } + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } + + /** + * Starts a data source level ingest progress bar for this job. + */ + private void startDataSourceIngestProgressBar() { + if (this.runInteractively) { + synchronized (this.dataSourceIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + this.dataSource.getName()); + this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. What remains to be done is to find out whether + // the user wants to cancel only the currently executing + // data source ingest module or the entire ingest job. + DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); + String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); + JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); + if (panel.cancelAllDataSourceIngestModules()) { + DataSourceIngestJob.this.cancel(); + } else { + DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule(); + } + return true; + } + }); + this.dataSourceIngestProgress.start(); + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + } + + /** + * Starts the file level ingest progress bar for this job. + */ + private void startFileIngestProgressBar() { + if (this.runInteractively) { + synchronized (this.fileIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. + DataSourceIngestJob.this.cancel(); + return true; + } + }); + this.fileIngestProgress.start(); + this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + } + } + } + + /** + * Checks to see if the ingest tasks for the current stage of this job are + * completed and does a stage transition if they are. + */ + private void checkForStageCompleted() { + synchronized (this.stageCompletionCheckLock) { + if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) { + switch (this.stage) { + case FIRST: + this.finishFirstStage(); + break; + case SECOND: + this.finish(); + break; + } + } + } + } + + /** + * Shuts down the first stage ingest pipelines and progress bars for this + * job and starts the second stage, if appropriate. + */ + private void finishFirstStage() { + // Shut down the file ingest pipelines. Note that no shut down is + // required for the data source ingest pipeline because data source + // ingest modules do not have a shutdown() method. + List errors = new ArrayList<>(); + while (!this.fileIngestPipelinesQueue.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); + errors.addAll(pipeline.shutDown()); + } + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + if (this.runInteractively) { + // Finish the first stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + + // Finish the file ingest progress bar, if it hasn't already + // been finished. + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + this.fileIngestProgress.finish(); + this.fileIngestProgress = null; + } + } + } + + /** + * Start the second stage, if appropriate. + */ + if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } else { + this.finish(); + } + } + + /** + * Shuts down the ingest pipelines and progress bars for this job. + */ + private void finish() { + this.stage = DataSourceIngestJob.Stages.FINALIZATION; + + if (this.runInteractively) { + // Finish the second stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + } + + this.parentJob.dataSourceJobFinished(this); + } + + /** + * Passes the data source for this job through the currently active data + * source level ingest pipeline. + * + * @param task A data source ingest task wrapping the data source. + */ + void process(DataSourceIngestTask task) { + try { + synchronized (this.dataSourceIngestPipelineLock) { + if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + List errors = new ArrayList<>(); + errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } + + if (this.runInteractively) { + /** + * Shut down the data source ingest progress bar right away. + * Data source-level processing is finished for this stage. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + } + + } finally { + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Passes a file from the data source for this job through the file level + * ingest pipeline. + * + * @param task A file ingest task. + * @throws InterruptedException if the thread executing this code is + * interrupted while blocked on taking from or putting to the file ingest + * pipelines collection. + */ + void process(FileIngestTask task) throws InterruptedException { + try { + if (!this.isCancelled()) { + FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); + if (!pipeline.isEmpty()) { + AbstractFile file = task.getFile(); + + synchronized (this.fileIngestProgressLock) { + ++this.processedFiles; + if (this.runInteractively) { + /** + * Update the file ingest progress bar. + */ + if (this.processedFiles <= this.estimatedFilesToProcess) { + this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); + } else { + this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); + } + this.filesInProgress.add(file.getName()); + } + } + + /** + * Run the file through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(pipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + if (this.runInteractively && !this.cancelled) { + synchronized (this.fileIngestProgressLock) { + /** + * Update the file ingest progress bar again, in + * case the file was being displayed. + */ + this.filesInProgress.remove(file.getName()); + if (this.filesInProgress.size() > 0) { + this.fileIngestProgress.progress(this.filesInProgress.get(0)); + } else { + this.fileIngestProgress.progress(""); + } + } + } + } + this.fileIngestPipelinesQueue.put(pipeline); + } + } finally { + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Adds more files from the data source for this job to the job, i.e., adds + * extracted or carved files. Not currently supported for the second stage + * of the job. + * + * @param files A list of the files to add. + */ + void addFiles(List files) { + if (DataSourceIngestJob.Stages.FIRST == this.stage) { // RJCTODO: Is this synchronized correctly + for (AbstractFile file : files) { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); + } + } else { + DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + } + + /** + * The intended clients of this method are ingest modules running code + * on an ingest thread that is holding a reference to an ingest task, in + * which case a completion check would not be necessary, so this is a + * bit of defensive programming. + */ + this.checkForStageCompleted(); + } + + /** + * Updates the display name shown on the current data source level ingest + * progress bar for this job. + * + * @param displayName The new display name. + */ + void updateDataSourceIngestProgressBarDisplayName(String displayName) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.setDisplayName(displayName); + } + } + } + + /** + * Switches the data source level ingest progress bar for this job to + * determinate mode. This should be called if the total work units to + * process the data source is known. + * + * @param workUnits Total number of work units for the processing of the + * data source. + */ + void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToDeterminate(workUnits); + } + } + } + } + + /** + * Switches the data source level ingest progress bar for this job to + * indeterminate mode. This should be called if the total work units to + * process the data source is unknown. + */ + void switchDataSourceIngestProgressBarToIndeterminate() { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + } + } + + /** + * Updates the data source level ingest progress bar for this job with the + * number of work units performed, if in the determinate mode. + * + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(int workUnits) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress("", workUnits); + } + } + } + } + + /** + * Updates the data source level ingest progress for this job with a new + * task name, where the task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + */ + void advanceDataSourceIngestProgressBar(String currentTask) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress(currentTask); + } + } + } + } + + /** + * Updates the data source level ingest progress bar for this with a new + * task name and the number of work units performed, if in the determinate + * mode. The task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.dataSourceIngestProgress.progress(currentTask, workUnits); + } + } + } + + /** + * Queries whether or not a temporary cancellation of data source level + * ingest in order to stop the currently executing data source level ingest + * module is in effect for this job. + * + * @return True or false. + */ + boolean currentDataSourceIngestModuleIsCancelled() { + return this.currentDataSourceIngestModuleCancelled; + } + + /** + * Rescind a temporary cancellation of data source level ingest that was + * used to stop a single data source level ingest module fro this job. + */ + void currentDataSourceIngestModuleCancellationCompleted() { + this.currentDataSourceIngestModuleCancelled = false; + + if (this.runInteractively) { + /** + * A new progress bar must be created because the cancel button of + * the previously constructed component is disabled by NetBeans when + * the user selects the "OK" button of the cancellation confirmation + * dialog popped up by NetBeans when the progress bar cancel button + * is pressed. + */ + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + this.startDataSourceIngestProgressBar(); + } + } + } + + /** + * Gets the currently running data source level ingest module for this job. + * + * @return The currently running module, may be null. + */ + DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { + if (null != this.currentDataSourceIngestPipeline) { + return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); + } else { + return null; + } + } + + /** + * Requests a temporary cancellation of data source level ingest for this + * job in order to stop the currently executing data source ingest module. + */ + void cancelCurrentDataSourceIngestModule() { + this.currentDataSourceIngestModuleCancelled = true; + } + + /** + * Requests cancellation of ingest, i.e., a shutdown of the data source + * level and file level ingest pipelines. + */ + void cancel() { + if (this.runInteractively) { + /** + * Put a cancellation message on data source level ingest progress + * bar, if it is still running. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (dataSourceIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + dataSource.getName()); + dataSourceIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), + "IngestJob.progress.cancelling", + displayName)); + } + } + + /** + * Put a cancellation message on the file level ingest progress bar, + * if it is still running. + */ + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", + displayName)); + } + } + } + + this.cancelled = true; + + /** + * Tell the task scheduler to cancel all pending tasks, i.e., tasks not + * not being performed by an ingest thread. + */ + DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); + this.checkForStageCompleted(); + } + + /** + * Queries whether or not cancellation, i.e., a shutdown of the data source + * level and file level ingest pipelines for this job, has been requested. + * + * @return True or false. + */ + boolean isCancelled() { + return this.cancelled; + } + + /** + * Write ingest module errors to the log. + * + * @param errors The errors. + */ + private void logIngestModuleErrors(List errors) { + for (IngestModuleError error : errors) { + DataSourceIngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS + } + } + + /** + * Gets a snapshot of this jobs state and performance. + * + * @return An ingest job statistics object. + */ + Snapshot getSnapshot() { + return new Snapshot(); + } + + /** + * Stores basic diagnostic statistics for a data source ingest job. + */ + class Snapshot { + + private final String dataSource; + private final long jobId; + private final long jobStartTime; + private final long snapShotTime; + private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule; + private boolean fileIngestRunning; + private Date fileIngestStartTime; + private final long processedFiles; + private final long estimatedFilesToProcess; + private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; + + /** + * Constructs an object to store basic diagnostic statistics for a data + * source ingest job. + */ + Snapshot() { + this.dataSource = DataSourceIngestJob.this.dataSource.getName(); + this.jobId = DataSourceIngestJob.this.id; + this.jobStartTime = DataSourceIngestJob.this.createTime; + this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule(); + + /** + * Determine whether file ingest is running at the time of this + * snapshot and determine the earliest file ingest level pipeline + * start time, if file ingest was started at all. + */ + for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) { + if (pipeline.isRunning()) { + this.fileIngestRunning = true; + } + Date pipelineStartTime = pipeline.getStartTime(); + if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(this.fileIngestStartTime))) { + this.fileIngestStartTime = pipelineStartTime; + } + } + + /** + * Get processed file statistics. + */ + synchronized (DataSourceIngestJob.this.fileIngestProgressLock) { + this.processedFiles = DataSourceIngestJob.this.processedFiles; + this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess; + this.snapShotTime = new Date().getTime(); + } + + /** + * Get a snapshot of the tasks currently in progress for this job. + */ + this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); + } + + /** + * Gets time these statistics were collected. + * + * @return The statistics collection time as number of milliseconds + * since January 1, 1970, 00:00:00 GMT. + */ + long getSnapshotTime() { + return snapShotTime; + } + + /** + * Gets the name of the data source associated with the ingest job that + * is the subject of this snapshot. + * + * @return A data source name string. + */ + String getDataSource() { + return dataSource; + } + + /** + * Gets the identifier of the ingest job that is the subject of this + * snapshot. + * + * @return The ingest job id. + */ + long getJobId() { + return this.jobId; + } + + /** + * Gets the time the ingest job was started. + * + * @return The start time as number of milliseconds since January 1, + * 1970, 00:00:00 GMT. + */ + long getJobStartTime() { + return jobStartTime; + } + + DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() { + return this.dataSourceLevelIngestModule; + } + + boolean fileIngestIsRunning() { + return this.fileIngestRunning; + } + + Date fileIngestStartTime() { + return this.fileIngestStartTime; + } + + /** + * Gets files per second throughput since the ingest job that is the + * subject of this snapshot started. + * + * @return Files processed per second (approximate). + */ + double getSpeed() { + return (double) processedFiles / ((snapShotTime - jobStartTime) / 1000); + } + + /** + * Gets the number of files processed for the job so far. + * + * @return The number of processed files. + */ + long getFilesProcessed() { + return processedFiles; + } + + /** + * Gets an estimate of the files that still need to be processed for + * this job. + * + * @return The estimate. + */ + long getFilesEstimated() { + return estimatedFilesToProcess; + } + + long getRootQueueSize() { + return this.tasksSnapshot.getRootQueueSize(); + } + + long getDirQueueSize() { + return this.tasksSnapshot.getDirectoryTasksQueueSize(); + } + + long getFileQueueSize() { + return this.tasksSnapshot.getFileQueueSize(); + } + + long getDsQueueSize() { + return this.tasksSnapshot.getDsQueueSize(); + } + + long getRunningListSize() { + return this.tasksSnapshot.getRunningListSize(); + } + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index bde810e21d..3c0a3361da 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -25,9 +25,9 @@ import org.netbeans.api.progress.ProgressHandle; */ public class DataSourceIngestModuleProgress { - private final IngestJob job; + private final DataSourceIngestJob job; - DataSourceIngestModuleProgress(IngestJob job) { + DataSourceIngestModuleProgress(DataSourceIngestJob job) { this.job = job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 54d06023ee..0838abe2bc 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -19,40 +19,61 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source ingest modules. It starts them, - * shuts them down, and runs them in sequential order. + * This class manages a sequence of data source level ingest modules for a data + * source ingest job. It starts the modules, runs data sources through them, and + * shuts them down when data source level ingest is complete. + *

+ * This class is thread-safe. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; - private final List modules = new ArrayList<>(); + private final DataSourceIngestJob job; + private final List modules = new ArrayList<>(); + private volatile PipelineModule currentModule; - DataSourceIngestPipeline(IngestJob job, List moduleTemplates) { + /** + * Constructs an object that manages a sequence of data source level ingest + * modules. It starts the modules, runs data sources through them, and shuts + * them down when data source level ingest is complete. + * + * @param job The data source ingest job that owns this pipeline. + * @param moduleTemplates Templates for the creating the ingest modules that + * make up this pipeline. + */ + DataSourceIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - - // Create an ingest module instance from each data source ingest module - // template. for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); modules.add(module); } - } + } } + /** + * Indicates whether or not there are any ingest modules in this pipeline. + * + * @return True or false. + */ boolean isEmpty() { return modules.isEmpty(); } - List startUp() { + /** + * Starts up the ingest modules in this pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ + synchronized List startUp() { List errors = new ArrayList<>(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall @@ -62,57 +83,118 @@ final class DataSourceIngestPipeline { return errors; } - List process(DataSourceIngestTask task) { + /** + * Runs a data source through the ingest modules in sequential order. + * + * @param task A data source level ingest task containing a data source to + * be processed. + * @return A list of processing errors, possible empty. + */ + synchronized List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { + this.currentModule = module; String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", module.getDisplayName(), dataSource.getName()); this.job.updateDataSourceIngestProgressBarDisplayName(displayName); this.job.switchDataSourceIngestProgressBarToIndeterminate(); - ingestManager.setIngestTaskProgress(task, module.getDisplayName()); + DataSourceIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(dataSource, new DataSourceIngestModuleProgress(this.job)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } if (this.job.isCancelled()) { break; - } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { + } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { this.job.currentDataSourceIngestModuleCancellationCompleted(); } } + this.currentModule = null; ingestManager.setIngestTaskProgressCompleted(task); return errors; } - private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + /** + * Gets the currently running module. + * + * @return The module, possibly null if no module is currently running. + */ + PipelineModule getCurrentlyRunningModule() { + return this.currentModule; + } + + /** + * This class decorates a data source level ingest module with a display + * name and a processing start time. + */ + static class PipelineModule implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; + private volatile Date processingStartTime; - DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { + /** + * Constructs an object that decorates a data source level ingest module + * with a display name and a processing start time. + * + * @param module The data source level ingest module to be decorated. + * @param displayName The display name. + */ + PipelineModule(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; + this.processingStartTime = new Date(); } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { - return module.getClass().getCanonicalName(); + return this.module.getClass().getCanonicalName(); } + /** + * Gets the display of the decorated ingest module. + * + * @return The display name. + */ String getDisplayName() { - return displayName; + return this.displayName; } + /** + * Gets the time the decorated ingest module started processing the data + * source. + * + * @return The start time, will be null if the module has not started + * processing the data source yet. + */ + Date getProcessingStartTime() { + return this.processingStartTime; + } + + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); + this.module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - return module.process(dataSource, statusHelper); + this.processingStartTime = new Date(); + return this.module.process(dataSource, statusHelper); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index 4fecebbe84..66fa744682 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.ingest; final class DataSourceIngestTask extends IngestTask { - DataSourceIngestTask(IngestJob job) { + DataSourceIngestTask(DataSourceIngestJob job) { super(job); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java index 369e34925c..42dead516c 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java @@ -38,10 +38,10 @@ public interface FileIngestModule extends IngestModule { /** * Invoked by Autopsy when an ingest job is completed (either because the * data has been analyzed or because the job was canceled - check - * IngestJobContext.isJobCancelled()), before the ingest module instance is - * discarded. The module should respond by doing things like releasing - * private resources, submitting final results, and posting a final ingest - * message. + * IngestJobContext.fileIngestIsCancelled()), before the ingest module + * instance is discarded. The module should respond by doing things like + * releasing private resources, submitting final results, and posting a + * final ingest message. */ void shutDown(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 329c717d5c..78db0b482d 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -19,43 +19,82 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * This class manages a sequence of file ingest modules. It starts them, - * shuts them down, and runs a file through them. + * This class manages a sequence of file level ingest modules for a data source + * ingest job. It starts the modules, runs files through them, and shuts them + * down when file level ingest is complete. + *

+ * This class is thread-safe. */ final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; - private final List modules = new ArrayList<>(); + private final DataSourceIngestJob job; + private final List modules = new ArrayList<>(); + private Date startTime; + private volatile boolean running; - FileIngestPipeline(IngestJob job, List moduleTemplates) { + /** + * Constructs an object that manages a sequence of file level ingest + * modules. It starts the modules, runs files through them, and shuts them + * down when file level ingest is complete. + * + * @param job The data source ingest job that owns the pipeline. + * @param moduleTemplates The ingest module templates that define the + * pipeline. + */ + FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - - // Create an ingest module instance from each file ingest module - // template. for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { - FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); modules.add(module); } } } + /** + * Queries whether or not there are any ingest modules in this pipeline. + * + * @return True or false. + */ boolean isEmpty() { return this.modules.isEmpty(); } /** - * Start up all of the modules in the pipeline. - * @return List of errors or empty list if no errors + * Queries whether or not this pipeline is running. + * + * @return True or false. */ - List startUp() { + boolean isRunning() { + return this.running; + } + + /** + * Returns the start up time of this pipeline. + * + * @return The file processing start time, may be null if this pipeline has + * not been started yet. + */ + Date getStartTime() { + return this.startTime; + } + + /** + * Starts up all of the ingest modules in the pipeline. + * + * @return List of start up errors, possibly empty. + */ + synchronized List startUp() { + this.startTime = new Date(); + this.running = true; List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall @@ -66,16 +105,15 @@ final class FileIngestPipeline { } /** - * Process the file down the pipeline of modules. - * Startup must have been called before this is called. - * - * @param file File to analyze - * @return List of errors or empty list if no errors + * Runs a file through the ingest modules in sequential order. + * + * @param task A file level ingest task containing a file to be processed. + * @return A list of processing errors, possible empty. */ - List process(FileIngestTask task) { + synchronized List process(FileIngestTask task) { List errors = new ArrayList<>(); AbstractFile file = task.getFile(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(file); @@ -94,49 +132,86 @@ final class FileIngestPipeline { return errors; } - List shutDown() { + /** + * Shuts down all of the modules in the pipeline. + * + * @return A list of shut down errors, possibly empty. + */ + synchronized List shutDown() { List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { module.shutDown(); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } + this.running = false; return errors; } - private static final class FileIngestModuleDecorator implements FileIngestModule { + /** + * This class decorates a file level ingest module with a display name. + */ + private static final class PipelineModule implements FileIngestModule { private final FileIngestModule module; private final String displayName; - FileIngestModuleDecorator(FileIngestModule module, String displayName) { + /** + * Constructs an object that decorates a file level ingest module with a + * display name. + * + * @param module The file level ingest module to be decorated. + * @param displayName The display name. + */ + PipelineModule(FileIngestModule module, String displayName) { this.module = module; this.displayName = displayName; } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { return module.getClass().getCanonicalName(); } + /** + * Gets the display name of the decorated ingest module. + * + * @return The display name. + */ String getDisplayName() { return displayName; } + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(AbstractFile file) { return module.process(file); } + /** + * @inheritDoc + */ @Override public void shutDown() { module.shutDown(); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 6fa02cf4fc..41a4045b12 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -29,7 +29,7 @@ final class FileIngestTask extends IngestTask { private final AbstractFile file; - FileIngestTask(IngestJob job, AbstractFile file) { + FileIngestTask(DataSourceIngestJob job, AbstractFile file) { super(job); this.file = file; } @@ -53,8 +53,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestJob job = getIngestJob(); - IngestJob otherJob = other.getIngestJob(); + DataSourceIngestJob job = getIngestJob(); + DataSourceIngestJob otherJob = other.getIngestJob(); if (job != otherJob && (job == null || !job.equals(otherJob))) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index cd1e6e3480..8218186b4c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -19,1099 +19,312 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import javax.swing.JOptionPane; -import org.netbeans.api.progress.ProgressHandle; -import org.netbeans.api.progress.ProgressHandleFactory; -import org.openide.util.Cancellable; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** - * Encapsulates a data source to be processed and the settings, ingest module - * pipelines, and progress bars that are used to process it. + * Runs a collection of data sources through a set of ingest modules specified + * via ingest job settings. + *

+ * This class is thread-safe. */ -final class IngestJob { +public final class IngestJob { - private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); - - /** - * The task scheduler singleton is responsible for creating and scheduling - * the ingest tasks that make up ingest jobs. - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - - /** - * These static fields are used for the creation and management of ingest - * jobs in progress. - */ - private static volatile boolean jobCreationIsEnabled; - private static final AtomicLong nextJobId = new AtomicLong(0L); - private static final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); - - /** - * These fields define the ingest job, including its ingest pipelines. Note - * that there is a collection of multiple copies of the file ingest - * pipeline, one for each file ingest thread. - */ + private static final AtomicLong nextId = new AtomicLong(0L); private final long id; - private final Content dataSource; - private final IngestJobSettings ingestJobSettings; - private final Object dataSourceIngestPipelineLock; - private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; - private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; - private final LinkedBlockingQueue fileIngestPipelines; + private final Map dataSourceJobs; + private boolean cancelled; /** - * An ingest runs in stages. - */ - private static enum Stages { - - /** - * Setting up for processing. - */ - INITIALIZATION, - /** - * Running high priority data source level ingest modules and file level - * ingest modules. - */ - FIRST, - /** - * Running lower priority, usually long-running, data source level - * ingest modules. - */ - SECOND, - /** - * Cleaning up. - */ - FINALIZATION - }; - private Stages stage; - private final Object stageCompletionCheckLock; - - /** - * These fields are used to provide data source level task progress bars for - * the job. - */ - private final Object dataSourceIngestProgressLock; - private ProgressHandle dataSourceIngestProgress; - - /** - * These fields are used to provide file level ingest task progress bars for - * the job. - */ - private final Object fileIngestProgressLock; - private final List filesInProgress; - private long estimatedFilesToProcess; - private long processedFiles; - private ProgressHandle fileIngestProgress; - - /** - * These fields support cancellation of either the currently running data - * source level ingest module or the entire ingest job. - */ - private volatile boolean currentDataSourceIngestModuleCancelled; - private volatile boolean cancelled; - - /** - * This field is used for generating ingest job diagnostic data. - */ - private final long startTime; - - /** - * Enables and disables ingest job creation. + * Constructs an ingest job that runs a collection of data sources through a + * set of ingest modules specified via ingest job settings. * - * @param enabled True or false. + * @param dataSources The data sources to be ingested. + * @param settings The ingest job settings. + * @param runInteractively Whether or not this job should use progress bars, + * message boxes for errors, etc. */ - static void jobCreationEnabled(boolean enabled) { - IngestJob.jobCreationIsEnabled = enabled; - } - - /** - * Starts an ingest job for a data source. - * - * @param dataSource The data source to ingest. - * @param settings The settings for the job. - * @return A collection of ingest module start up errors, empty on success. - */ - static List startJob(Content dataSource, IngestJobSettings settings) { - List errors = new ArrayList<>(); - if (IngestJob.jobCreationIsEnabled) { - long jobId = nextJobId.incrementAndGet(); - IngestJob job = new IngestJob(jobId, dataSource, settings); - IngestJob.jobsById.put(jobId, job); - errors = job.start(settings.getEnabledIngestModuleTemplates()); - if (errors.isEmpty() && job.hasIngestPipeline()) { - IngestManager.getInstance().fireIngestJobStarted(jobId); - IngestJob.logger.log(Level.INFO, "Ingest job {0} started", jobId); - } else { - IngestJob.jobsById.remove(jobId); - } - } - return errors; - } - - /** - * Queries whether or not ingest jobs are running. - * - * @return True or false. - */ - static boolean ingestJobsAreRunning() { - return !jobsById.isEmpty(); - } - - /** - * Gets snapshots of the state of all running ingest jobs. - * - * @return A list of ingest job state snapshots. - */ - static List getJobSnapshots() { - List snapShots = new ArrayList<>(); - for (IngestJob job : IngestJob.jobsById.values()) { - snapShots.add(job.getSnapshot()); - } - return snapShots; - } - - /** - * Cancels all running ingest jobs. - */ - static void cancelAllJobs() { - for (IngestJob job : jobsById.values()) { - job.cancel(); + IngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { + this.id = IngestJob.nextId.getAndIncrement(); + this.dataSourceJobs = new HashMap<>(); + for (Content dataSource : dataSources) { + DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, runInteractively); + this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); } } /** - * Constructs an ingest job. - * - * @param id The identifier assigned to the job. - * @param dataSource The data source to be ingested. - * @param processUnallocatedSpace Whether or not unallocated space should be - * processed during the ingest job. - */ - private IngestJob(long id, Content dataSource, IngestJobSettings settings) { - this.id = id; - this.dataSource = dataSource; - - this.ingestJobSettings = settings; - this.dataSourceIngestPipelineLock = new Object(); - this.fileIngestPipelines = new LinkedBlockingQueue<>(); - this.filesInProgress = new ArrayList<>(); - this.dataSourceIngestProgressLock = new Object(); - this.fileIngestProgressLock = new Object(); - this.stage = IngestJob.Stages.INITIALIZATION; - this.stageCompletionCheckLock = new Object(); - this.startTime = new Date().getTime(); - } - - /** - * Gets the identifier assigned to this job. + * Gets the unique identifier assigned to this ingest job. * * @return The job identifier. */ - long getId() { + public long getId() { return this.id; } /** - * Gets the data source to be ingested by this job. - * - * @return A Content object representing the data source. - */ - Content getDataSource() { - return this.dataSource; - } - - /** - * Gets whether or not unallocated space should be processed as part of this - * job. + * Checks to see if this ingest job has at least one ingest pipeline when + * its settings are applied. * * @return True or false. */ - boolean shouldProcessUnallocatedSpace() { - return this.ingestJobSettings.getProcessUnallocatedSpace(); + synchronized boolean hasIngestPipeline() { + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + if (dataSourceJob.hasIngestPipeline()) { + return true; + } + } + return false; } /** - * Passes the data source for this job through the currently active data - * source level ingest pipeline. + * Starts this ingest job by starting its ingest module pipelines and + * scheduling the ingest tasks that make up the job. * - * @param task A data source ingest task wrapping the data source. + * @return A collection of ingest module start up errors, empty on success. */ - void process(DataSourceIngestTask task) { - try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - /** - * Run the data source through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } + synchronized List start() { + List errors = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + errors.addAll(dataSourceJob.start()); + if (!errors.isEmpty()) { + // RJCTODO: Need to let sucessfully started data source ingest + // jobs know they should shut down. + break; + } + } + return errors; + } + + /** + * Gets a snapshot of the progress of this ingest job. + * + * @return The snapshot. + */ + synchronized public ProgressSnapshot getSnapshot() { + DataSourceIngestModuleHandle moduleHandle = null; + boolean fileIngestIsRunning = false; + Date fileIngestStartTime = null; + for (DataSourceIngestJob.Snapshot snapshot : this.getDataSourceIngestJobSnapshots()) { + if (null == moduleHandle) { + DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); + if (null != module) { + moduleHandle = new DataSourceIngestModuleHandle(this.dataSourceJobs.get(snapshot.getJobId()), module); } } - - /** - * Shut down the data source ingest progress bar right away. Data - * source-level processing is finished for this stage. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } + if (snapshot.fileIngestIsRunning()) { + fileIngestIsRunning = true; + } + Date childFileIngestStartTime = snapshot.fileIngestStartTime(); + if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) { + fileIngestStartTime = childFileIngestStartTime; } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); } + return new ProgressSnapshot(moduleHandle, fileIngestIsRunning, fileIngestStartTime, this.cancelled); } /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. + * Gets snapshots of the progress of each of this ingest job's child data + * source ingest jobs. * - * @param task A file ingest task. - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or putting to the file ingest - * pipelines collection. + * @return A list of data source ingest job progress snapshots. */ - void process(FileIngestTask task) throws InterruptedException { - try { - if (!this.isCancelled()) { - /** - * Get a file ingest pipeline not currently in use by another - * file ingest thread. - */ - FileIngestPipeline pipeline = this.fileIngestPipelines.take(); - if (!pipeline.isEmpty()) { - /** - * Get the file to process. - */ - AbstractFile file = task.getFile(); - - /** - * Update the file ingest progress bar. - */ - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); - } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); - } - this.filesInProgress.add(file.getName()); - } - - /** - * Run the file through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - /** - * Update the file ingest progress bar again, in case the - * file was being displayed. - */ - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); - } else { - this.fileIngestProgress.progress(""); - } - } - } - } - - /** - * Relinquish the pipeline so it can be reused by another file - * ingest thread. - */ - this.fileIngestPipelines.put(pipeline); - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); + synchronized List getDataSourceIngestJobSnapshots() { + List snapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + snapshots.add(dataSourceJob.getSnapshot()); } + return snapshots; } /** - * Adds more files to an ingest job, i.e., extracted or carved files. Not - * currently supported for the second stage of the job. - * - * @param files A list of files to add. + * Requests cancellation of this ingest job, which means discarding + * unfinished tasks and stopping the ingest pipelines. Returns immediately, + * but there may be a delay before all of the ingest modules in the + * pipelines respond by stopping processing. */ - void addFiles(List files) { - /** - * Note: This implementation assumes that this is being called by an an - * ingest module running code on an ingest thread that is holding a - * reference to an ingest task, so no task completion check is done. - */ - if (IngestJob.Stages.FIRST == this.stage) { - for (AbstractFile file : files) { - IngestJob.taskScheduler.scheduleFileIngestTask(this, file); - } - } else { - IngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + synchronized public void cancel() { + for (DataSourceIngestJob job : this.dataSourceJobs.values()) { + job.cancel(); } - - /** - * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a task completion check would not be necessary. This is a - * bit of defensive programming. - */ - this.checkForStageCompleted(); - } - - /** - * Updates the display name of the data source level ingest progress bar. - * - * @param displayName The new display name. - */ - void updateDataSourceIngestProgressBarDisplayName(String displayName) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); - } - } - } - - /** - * Switches the data source level ingest progress bar to determinate mode. - * This should be called if the total work units to process the data source - * is known. - * - * @param workUnits Total number of work units for the processing of the - * data source. - */ - void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); - } - } - } - } - - /** - * Switches the data source level ingest progress bar to indeterminate mode. - * This should be called if the total work units to process the data source - * is unknown. - */ - void switchDataSourceIngestProgressBarToIndeterminate() { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with the number of work - * units performed, if in the determinate mode. - * - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); - } - } - } - } - - /** - * Updates the data source level ingest progress with a new task name, where - * the task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - */ - void advanceDataSourceIngestProgressBar(String currentTask) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with a new task name - * and the number of work units performed, if in the determinate mode. The - * task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); - } - } - } - - /** - * Queries whether or not a temporary cancellation of data source level - * ingest in order to stop the currently executing data source level ingest - * module is in effect. - * - * @return True or false. - */ - boolean currentDataSourceIngestModuleIsCancelled() { - return this.currentDataSourceIngestModuleCancelled; - } - - /** - * Rescind a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module. - */ - void currentDataSourceIngestModuleCancellationCompleted() { - this.currentDataSourceIngestModuleCancelled = false; - - /** - * A new progress bar must be created because the cancel button of the - * previously constructed component is disabled by NetBeans when the - * user selects the "OK" button of the cancellation confirmation dialog - * popped up by NetBeans when the progress bar cancel button was - * pressed. - */ - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - this.startDataSourceIngestProgressBar(); - } - } - - /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. - */ - void cancel() { - /** - * Put a cancellation message on data source level ingest progress bar, - * if it is still running. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (dataSourceIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - dataSource.getName()); - dataSourceIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), - "IngestJob.progress.cancelling", - displayName)); - } - } - - /** - * Put a cancellation message on the file level ingest progress bar, if - * it is still running. - */ - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", - displayName)); - } - } - this.cancelled = true; - - /** - * Tell the task scheduler to cancel all pending tasks, i.e., tasks not - * not being performed by an ingest thread. - */ - IngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); - this.checkForStageCompleted(); } /** - * Queries whether or not cancellation of ingest i.e., a shutdown of the - * data source level and file level ingest pipelines, has been requested. + * Queries whether or not cancellation of this ingest job has been + * requested. * * @return True or false. */ - boolean isCancelled() { + synchronized public boolean isCancelled() { return this.cancelled; } /** - * Starts up the ingest pipelines and ingest progress bars. + * Provides a callback for completed data source ingest jobs, allowing this + * ingest job to notify the ingest manager when it is complete. * - * @return A collection of ingest module startup errors, empty on success. + * @param dataSourceIngestJob A completed data source ingest job. */ - private List start(List ingestModuleTemplates) { - this.createIngestPipelines(ingestModuleTemplates); - List errors = startUpIngestPipelines(); - if (errors.isEmpty()) { - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - this.startFirstStage(); - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } + synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { + this.dataSourceJobs.remove(dataSourceIngestJob.getId()); + if (this.dataSourceJobs.isEmpty()) { + IngestManager.getInstance().finishIngestJob(this); } - return errors; } /** - * Creates the file and data source ingest pipelines. - * - * @param ingestModuleTemplates Ingest module templates to use to populate - * the pipelines. + * A snapshot of the progress of an ingest job. */ - private void createIngestPipelines(List ingestModuleTemplates) { + public static final class ProgressSnapshot { + + private final DataSourceIngestModuleHandle dataSourceModule; + private final boolean fileIngestRunning; + private final Date fileIngestStartTime; + private final boolean cancelled; + /** - * Make mappings of ingest module factory class names to templates. + * Constructs a snapshot of ingest job progress. + * + * @param dataSourceModule The currently running data source level + * ingest module, may be null. + * @param fileIngestRunning Whether or not file ingest is currently + * running. + * @param fileIngestStartTime The start time of file level ingest, may + * be null. + * @param cancelled Whether or not a cancellation request has been + * issued. */ - Map dataSourceModuleTemplates = new HashMap<>(); - Map fileModuleTemplates = new HashMap<>(); - for (IngestModuleTemplate template : ingestModuleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } - if (template.isFileIngestModuleTemplate()) { - fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } + private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) { + this.dataSourceModule = dataSourceModule; + this.fileIngestRunning = fileIngestRunning; + this.fileIngestStartTime = fileIngestStartTime; + this.cancelled = cancelled; } /** - * Use the mappings and the ingest pipelines configuration to create - * ordered lists of ingest module templates for each ingest pipeline. + * Gets a handle to the currently running data source level ingest + * module at the time the snapshot was taken. + * + * @return The handle, may be null. */ - IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); - - /** - * Add any module templates that were not specified in the pipelines - * configuration to an appropriate pipeline - either the first stage - * data source ingest pipeline or the file ingest pipeline. - */ - for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { - firstStageDataSourceModuleTemplates.add(template); - } - for (IngestModuleTemplate template : fileModuleTemplates.values()) { - fileIngestModuleTemplates.add(template); + public DataSourceIngestModuleHandle runningDataSourceIngestModule() { + return this.dataSourceModule; } /** - * Construct the data source ingest pipelines. + * Queries whether or not file level ingest was running at the time the + * snapshot was taken. + * + * @return True or false. */ - this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); + public boolean fileIngestIsRunning() { + return this.fileIngestRunning; + } /** - * Construct the file ingest pipelines, one per file ingest thread. + * Gets the time that file level ingest started. + * + * @return The start time, may be null. */ - try { - int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); - } - } catch (InterruptedException ex) { + public Date fileIngestStartTime() { + return new Date(this.fileIngestStartTime.getTime()); + } + + /** + * Queries whether or not a cancellation request had been issued at the + * time the snapshot was taken. + * + * @return True or false. + */ + public boolean isCancelled() { + return this.cancelled; + } + + } + + /** + * A handle to a data source level ingest module that can be used to get + * basic information about the module and to request cancellation of the + * module. + */ + public static class DataSourceIngestModuleHandle { + + private final DataSourceIngestJob job; + private final DataSourceIngestPipeline.PipelineModule module; + private final boolean cancelled; + + /** + * Constructs a handle to a data source level ingest module that can be + * used to get basic information about the module and to request + * cancellation of the module. + * + * @param DataSourceIngestJob The data source ingest job that owns the + * data source level ingest module. + * @param module The data source level ingest module. + */ + private DataSourceIngestModuleHandle(DataSourceIngestJob job, DataSourceIngestPipeline.PipelineModule module) { + this.job = job; + this.module = module; + this.cancelled = job.currentDataSourceIngestModuleIsCancelled(); + } + + /** + * Gets the display name of the data source level ingest module + * associated with this handle. + * + * @return The display name. + */ + public String displayName() { + return this.module.getDisplayName(); + } + + /** + * Gets the time the data source level ingest module associated with + * this handle began processing. + * + * @return The module processing start time. + */ + public Date startTime() { + return this.module.getProcessingStartTime(); + } + + /** + * Queries whether or not cancellation of the data source level ingest + * module associated with this handle has been requested. + * + * @return True or false. + */ + public boolean isCancelled() { + return this.cancelled; + } + + /** + * Requests cancellation of the ingest module associated with this + * handle. Returns immediately, but there may be a delay before the + * ingest module responds by stopping processing. + */ + public void cancel() { /** - * The current thread was interrupted while blocked on a full queue. - * Blocking should actually never happen here, but reset the - * interrupted flag rather than just swallowing the exception. + * TODO: Cancellation needs to be more precise. The long-term + * solution is to add a cancel() method to IngestModule and do away + * with the cancellation queries of IngestJobContext. However, until + * an API change is legal, a cancel() method can be added to the + * DataSourceIngestModuleAdapter and FileIngestModuleAdapter classes + * and an instanceof check can be used to call it, with this code as + * the default implementation and the fallback. All of the ingest + * modules participating in this workaround will need to consult the + * cancelled flag in the adapters. */ - Thread.currentThread().interrupt(); - } - } - - /** - * Use an ordered list of ingest module factory class names to create an - * ordered output list of ingest module templates for an ingest pipeline. - * The ingest module templates are removed from the input collection as they - * are added to the output collection. - * - * @param ingestModuleTemplates A mapping of ingest module factory class - * names to ingest module templates. - * @param pipelineConfig An ordered list of ingest module factory class - * names representing an ingest pipeline. - * @return - */ - private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { - List templates = new ArrayList<>(); - for (String moduleClassName : pipelineConfig) { - if (ingestModuleTemplates.containsKey(moduleClassName)) { - templates.add(ingestModuleTemplates.remove(moduleClassName)); + if (this.job.getCurrentDataSourceIngestModule() == this.module) { + this.job.cancelCurrentDataSourceIngestModule(); } } - return templates; - } - - /** - * Starts the first stage of the job. - */ - private void startFirstStage() { - this.stage = IngestJob.Stages.FIRST; - - /** - * Start one or both of the first stage ingest progress bars. - */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - IngestJob.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - IngestJob.taskScheduler.scheduleFileIngestTasks(this); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never to check for completion of this stage of the job. - */ - this.checkForStageCompleted(); - } - } - - /** - * Starts the second stage of the ingest job. - */ - private void startSecondStage() { - this.stage = IngestJob.Stages.SECOND; - this.startDataSourceIngestProgressBar(); - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; - } - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } - - /** - * Checks to see if this job has at least one ingest pipeline. - * - * @return True or false. - */ - private boolean hasIngestPipeline() { - return this.hasFirstStageDataSourceIngestPipeline() - || this.hasFileIngestPipeline() - || this.hasSecondStageDataSourceIngestPipeline(); - } - - /** - * Checks to see if this job has a first stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasFirstStageDataSourceIngestPipeline() { - return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if this job has a second stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasSecondStageDataSourceIngestPipeline() { - return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if the job has a file level ingest pipeline. - * - * @return True or false. - */ - private boolean hasFileIngestPipeline() { - return (this.fileIngestPipelines.peek().isEmpty() == false); - } - - /** - * Starts up each of the file and data source level ingest modules to - * collect possible errors. - * - * @return A collection of ingest module startup errors, empty on success. - */ - private List startUpIngestPipelines() { - List errors = new ArrayList<>(); - - // Start up the first stage data source ingest pipeline. - errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); - - // Start up the second stage data source ingest pipeline. - errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); - - // Start up the file ingest pipelines (one per file ingest thread). - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { - errors.addAll(pipeline.startUp()); - if (!errors.isEmpty()) { - // If there are start up errors, the ingest job will not proceed - // and the errors will ultimately be reported to the user for - // possible remedy so shut down the pipelines now that an - // attempt has been made to start up the data source ingest - // pipeline and at least one copy of the file ingest pipeline. - // pipeline. There is no need to complete starting up all of the - // file ingest pipeline copies since any additional start up - // errors are likely redundant. - while (!this.fileIngestPipelines.isEmpty()) { - pipeline = this.fileIngestPipelines.poll(); - List shutDownErrors = pipeline.shutDown(); - if (!shutDownErrors.isEmpty()) { - logIngestModuleErrors(shutDownErrors); - } - } - break; - } - } - - logIngestModuleErrors(errors); - return errors; - } - - /** - * Starts the data source level ingest progress bar. - */ - private void startDataSourceIngestProgressBar() { - synchronized (this.dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. What remains to be done is to find out whether - // the user wants to cancel only the currently executing - // data source ingest module or the entire ingest job. - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { - IngestJob.this.cancel(); - } else { - IngestJob.this.cancelCurrentDataSourceIngestModule(); - } - return true; - } - }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - - /** - * Starts the file level ingest progress bar. - */ - private void startFileIngestProgressBar() { - synchronized (this.fileIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. - IngestJob.this.cancel(); - return true; - } - }); - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); - } - } - - /** - * Checks to see if the ingest tasks for the current stage are completed and - * does a stage transition if they are. - */ - private void checkForStageCompleted() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - switch (this.stage) { - case FIRST: - this.finishFirstStage(); - break; - case SECOND: - this.finish(); - break; - } - } - } - } - - /** - * Shuts down the first stage ingest pipelines and progress bars and starts - * the second stage, if appropriate. - */ - private void finishFirstStage() { - // Shut down the file ingest pipelines. Note that no shut down is - // required for the data source ingest pipeline because data source - // ingest modules do not have a shutdown() method. - List errors = new ArrayList<>(); - while (!this.fileIngestPipelines.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelines.poll(); - errors.addAll(pipeline.shutDown()); - } - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; - } - } - - /** - * Start the second stage, if appropriate. - */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } else { - this.finish(); - } - } - - /** - * Shuts down the ingest pipelines and progress bars for this job. - */ - private void finish() { - this.stage = IngestJob.Stages.FINALIZATION; - - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - IngestJob.jobsById.remove(this.id); - if (!this.isCancelled()) { - logger.log(Level.INFO, "Ingest job {0} completed", this.id); - IngestManager.getInstance().fireIngestJobCompleted(this.id); - } else { - logger.log(Level.INFO, "Ingest job {0} cancelled", this.id); - IngestManager.getInstance().fireIngestJobCancelled(this.id); - } - } - - /** - * Write ingest module errors to the log. - * - * @param errors The errors. - */ - private void logIngestModuleErrors(List errors) { - for (IngestModuleError error : errors) { - IngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS - } - } - - /** - * Requests a temporary cancellation of data source level ingest in order to - * stop the currently executing data source ingest module. - */ - private void cancelCurrentDataSourceIngestModule() { - this.currentDataSourceIngestModuleCancelled = true; - } - - /** - * Gets a snapshot of this jobs state and performance. - * - * @return An ingest job statistics object. - */ - private IngestJobSnapshot getSnapshot() { - return new IngestJobSnapshot(); - - } - - /** - * Stores basic diagnostic statistics for an ingest job. - */ - class IngestJobSnapshot { - - private final long jobId; - private final String dataSource; - private final long startTime; - private final long processedFiles; - private final long estimatedFilesToProcess; - private final long snapShotTime; - private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; - - /** - * Constructs an object to stores basic diagnostic statistics for an - * ingest job. - */ - IngestJobSnapshot() { - this.jobId = IngestJob.this.id; - this.dataSource = IngestJob.this.dataSource.getName(); - this.startTime = IngestJob.this.startTime; - synchronized (IngestJob.this.fileIngestProgressLock) { - this.processedFiles = IngestJob.this.processedFiles; - this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; - this.snapShotTime = new Date().getTime(); - } - - /** - * Get a snapshot of the tasks currently in progress for this job. - */ - this.tasksSnapshot = IngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); - } - - /** - * Gets the identifier of the ingest job that is the subject of this - * snapshot. - * - * @return The ingest job id. - */ - long getJobId() { - return this.jobId; - } - - /** - * Gets the name of the data source associated with the ingest job that - * is the subject of this snapshot. - * - * @return A data source name string. - */ - String getDataSource() { - return dataSource; - } - - /** - * Gets files per second throughput since the ingest job that is the - * subject of this snapshot started. - * - * @return Files processed per second (approximate). - */ - double getSpeed() { - return (double) processedFiles / ((snapShotTime - startTime) / 1000); - } - - /** - * Gets the time the ingest job was started. - * - * @return The start time as number of milliseconds since January 1, - * 1970, 00:00:00 GMT. - */ - long getStartTime() { - return startTime; - } - - /** - * Gets time these statistics were collected. - * - * @return The statistics collection time as number of milliseconds - * since January 1, 1970, 00:00:00 GMT. - */ - long getSnapshotTime() { - return snapShotTime; - } - - /** - * Gets the number of files processed for the job so far. - * - * @return The number of processed files. - */ - long getFilesProcessed() { - return processedFiles; - } - - /** - * Gets an estimate of the files that still need to be processed for - * this job. - * - * @return The estimate. - */ - long getFilesEstimated() { - return estimatedFilesToProcess; - } - - long getRootQueueSize() { - return this.tasksSnapshot.getRootQueueSize(); - } - - long getDirQueueSize() { - return this.tasksSnapshot.getDirectoryTasksQueueSize(); - } - - long getFileQueueSize() { - return this.tasksSnapshot.getFileQueueSize(); - } - - long getDsQueueSize() { - return this.tasksSnapshot.getDsQueueSize(); - } - - long getRunningListSize() { - return this.tasksSnapshot.getRunningListSize(); - } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java index ee1d65c115..78d9d7ef84 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java @@ -38,8 +38,8 @@ public final class IngestJobConfigurator { /** * Constructs an ingest job launcher that creates and persists ingest job - * settings for a particular context and launches ingest jobs that - * process one or more data sources using the settings. + * settings for a particular context and launches ingest jobs that process + * one or more data sources using the settings. * * @param context The context identifier. */ @@ -50,8 +50,8 @@ public final class IngestJobConfigurator { } /** - * Gets any warnings generated when the persisted ingest job settings - * for the specified context are loaded or saved. + * Gets any warnings generated when the persisted ingest job settings for + * the specified context are loaded or saved. * * @return A collection of warning messages, possibly empty. */ @@ -80,16 +80,13 @@ public final class IngestJobConfigurator { } /** - * Launches ingest jobs for one or more data sources using the ingest job + * Launches ingest job for one or more data sources using the ingest job * settings for the specified context. * * @param dataSources The data sources to ingest. */ @Deprecated public void startIngestJobs(List dataSources) { - IngestManager ingestManager = IngestManager.getInstance(); - for (Content dataSource : dataSources) { - ingestManager.startIngestJob(dataSource, this.settings, true); - } + IngestManager.getInstance().queueIngestJob(dataSources, this.settings); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 6587a20d19..020c05812b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -28,9 +28,9 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJobContext { - private final IngestJob ingestJob; + private final DataSourceIngestJob ingestJob; - IngestJobContext(IngestJob ingestJob) { + IngestJobContext(DataSourceIngestJob ingestJob) { this.ingestJob = ingestJob; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3a9a58a959..b0cb987710 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -18,10 +18,12 @@ */ package org.sleuthkit.autopsy.ingest; +import java.awt.EventQueue; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -46,33 +48,95 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** - * Manages the creation and execution of ingest jobs, i.e., processing of data - * sources by ingest modules. + * Manages the creation and execution of ingest jobs, i.e., the processing of + * data sources by ingest modules. */ public class IngestManager { + private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); + private static IngestManager instance; + + /** + * The ingest manager maintains a mapping of ingest job IDs to running + * ingest jobs. + */ + private final ConcurrentHashMap jobsById; + + /** + * Each runnable/callable task the ingest manager submits to its thread + * pools is given a unique thread/task ID. + */ + private final AtomicLong nextThreadId; + + /** + * Ingest jobs may be queued to be started on a pool thread by ingest job + * starters. A mapping of thread/task IDs to the result objects associated + * with each ingest job starter is maintained to provide handles that can be + * used to cancel the ingest job starter. + */ + private final ConcurrentHashMap> ingestJobStarters; + private final ExecutorService startIngestJobsThreadPool; + + /** + * Ingest jobs use an ingest task scheduler to break themselves down into + * data source level and file level tasks. The ingest scheduler puts these + * ingest tasks into queues for execution on ingest manager pool threads by + * ingest task executers. There is a single data source level ingest thread + * and a user configurable number of file level ingest threads. + */ + private final ExecutorService dataSourceIngestThreadPool; private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; - private static final int MAX_ERROR_MESSAGE_POSTS = 200; - private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); - private static IngestManager instance = null; - private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); - private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); - private final IngestMonitor ingestMonitor = new IngestMonitor(); - private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); - private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); + private int numberOfFileIngestThreads; private final ExecutorService fileIngestThreadPool; - private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); - private final AtomicLong nextThreadId = new AtomicLong(0L); - private final ConcurrentHashMap> startIngestJobsCallables = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. - private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); - private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. - private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); - private final Object processedFilesSnapshotLock = new Object(); - private ProcessedFilesSnapshot processedFilesSnapshot = new ProcessedFilesSnapshot(); + + /** + * The ingest manager uses the property change feature from Java Beans as an + * event publishing mechanism. There are two kinds of events, ingest job + * events and ingest module events. Property changes are fired by ingest + * event publishers on a pool thread. + */ + private final PropertyChangeSupport ingestJobEventPublisher; + private final PropertyChangeSupport ingestModuleEventPublisher; + private final ExecutorService fireIngestEventsThreadPool; + + /** + * The ingest manager uses an ingest monitor to determine when system + * resources are under pressure. If the monitor detects such a situation, it + * calls back to the ingest manager to cancel all ingest jobs in progress. + */ + private final IngestMonitor ingestMonitor; + + /** + * The ingest manager provides access to a top component that is used by + * ingest module to post messages for the user. A count of the posts is used + * as a cap to avoid bogging down the application. + */ + private static final int MAX_ERROR_MESSAGE_POSTS = 200; private volatile IngestMessageTopComponent ingestMessageBox; - private int numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; + private final AtomicLong ingestErrorMessagePosts; + + /** + * The ingest manager supports reporting of ingest processing progress by + * collecting snapshots of the activities of the ingest threads, ingest job + * progress, and ingest module run times. + */ + private final ConcurrentHashMap ingestThreadActivitySnapshots; + private final ConcurrentHashMap ingestModuleRunTimes; + + /** + * The ingest job creation capability of the ingest manager can be turned on + * and off to support an orderly shut down of the application. + */ + private volatile boolean jobCreationIsEnabled; + + /** + * The ingest manager can be directed to forgo use of message boxes, the + * ingest message box, NetBeans progress handles, etc. Running interactively + * is the default. + */ + private volatile boolean runInteractively; /** * Ingest job events. @@ -129,22 +193,158 @@ public class IngestManager { }; /** - * Gets the ingest manager. + * Gets the manager of the creation and execution of ingest jobs, i.e., the + * processing of data sources by ingest modules. * - * @return A singleton IngestManager object. + * @return A singleton ingest manager object. */ public synchronized static IngestManager getInstance() { if (instance == null) { - // Two stage construction to avoid allowing "this" reference to - // escape from the constructor via the property change listener. - // This is to ensure that a partially constructed ingest manager is - // not published to other threads. + /** + * Two stage construction to avoid allowing the "this" reference to + * be prematurely published from the constructor via the Case + * property change listener. + */ instance = new IngestManager(); instance.subscribeToCaseEvents(); } return instance; } + /** + * Constructs a manager of the creation and execution of ingest jobs, i.e., + * the processing of data sources by ingest modules. The manager immediately + * submits ingest task executers (Callable objects) to the data source level + * ingest and file level ingest thread pools. These ingest task executers + * are simple consumers that will normally run as long as the application + * runs. + */ + private IngestManager() { + this.runInteractively = true; + this.ingestModuleRunTimes = new ConcurrentHashMap<>(); + this.ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); + this.ingestErrorMessagePosts = new AtomicLong(0L); + this.ingestMonitor = new IngestMonitor(); + this.ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); + this.fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); + this.ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); + this.dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); + this.startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); + this.nextThreadId = new AtomicLong(0L); + this.jobsById = new ConcurrentHashMap<>(); + this.ingestJobStarters = new ConcurrentHashMap<>(); + + this.startDataSourceIngestThread(); + + numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); + if ((numberOfFileIngestThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfFileIngestThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { + numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; + UserPreferences.setNumberOfFileIngestThreads(numberOfFileIngestThreads); + } + fileIngestThreadPool = Executors.newFixedThreadPool(numberOfFileIngestThreads); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + startFileIngestThread(); + } + } + + /** + * Submits an ingest task executer Callable to the data source level ingest + * thread pool. + */ + private void startDataSourceIngestThread() { + long threadId = nextThreadId.incrementAndGet(); + dataSourceIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); + ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + } + + /** + * Submits a ingest task executer Callable to the file level ingest thread + * pool. + */ + private void startFileIngestThread() { + long threadId = nextThreadId.incrementAndGet(); + fileIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); + ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + } + + private void subscribeToCaseEvents() { + Case.addPropertyChangeListener(new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent event) { + if (event.getPropertyName().equals(Case.Events.CURRENT_CASE.toString())) { + if (event.getNewValue() != null) { + handleCaseOpened(); + } else { + handleCaseClosed(); + } + } + } + }); + } + + synchronized void handleCaseOpened() { + this.jobCreationIsEnabled = true; + clearIngestMessageBox(); + } + + synchronized void handleCaseClosed() { + this.jobCreationIsEnabled = false; + cancelAllIngestJobs(); + clearIngestMessageBox(); + } + + /** + * The ingest manager can be directed to forgo use of message boxes, the + * ingest message box, NetBeans progress handles, etc. Running interactively + * is the default. + * + * @param runInteractively whether or not to this ingest manager should run + * ingest interactively. + */ + public synchronized void setRunInteractively(boolean runInteractively) { + this.runInteractively = runInteractively; + } + + /** + * Called by the custom installer for this package once the window system is + * initialized, allowing the ingest manager to get the top component used to + * display ingest messages. + */ + void initIngestMessageInbox() { + ingestMessageBox = IngestMessageTopComponent.findInstance(); + } + + /** + * Post a message to the ingest messages in box. + * + * @param message The message to be posted. + */ + synchronized void postIngestMessage(IngestMessage message) { + if (ingestMessageBox != null && this.runInteractively) { + if (message.getMessageType() != IngestMessage.MessageType.ERROR && message.getMessageType() != IngestMessage.MessageType.WARNING) { + ingestMessageBox.displayMessage(message); + } else { + long errorPosts = ingestErrorMessagePosts.incrementAndGet(); + if (errorPosts <= MAX_ERROR_MESSAGE_POSTS) { + ingestMessageBox.displayMessage(message); + } else if (errorPosts == MAX_ERROR_MESSAGE_POSTS + 1) { + IngestMessage errorMessageLimitReachedMessage = IngestMessage.createErrorMessage( + NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.title"), + NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.subject"), + NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.msg", MAX_ERROR_MESSAGE_POSTS)); + ingestMessageBox.displayMessage(errorMessageLimitReachedMessage); + } + } + } + } + + private void clearIngestMessageBox() { + if (ingestMessageBox != null) { + ingestMessageBox.clearMessages(); + } + ingestErrorMessagePosts.set(0); + } + /** * Gets the number of file ingest threads the ingest manager will use to do * ingest jobs. @@ -156,48 +356,141 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for a data - * source. + * Queues an ingest job that will process a collection of data sources. The + * job will be started on a worker thread. * - * @param dataSource The data source to be processed. - * @param settings The ingest job settings. - * @param doStartupErrorsMsgBox Whether or not to display ingest module - * startup errors in a message box. + * @param dataSources The data sources to process. + * @param settings The settings for the ingest job. */ - public synchronized void startIngestJob(Content dataSource, IngestJobSettings settings, boolean doStartupErrorsMsgBox) { - if (!isIngestRunning()) { - clearIngestMessageBox(); + public synchronized void queueIngestJob(Collection dataSources, IngestJobSettings settings) { + if (this.jobCreationIsEnabled) { + IngestJob job = new IngestJob(dataSources, settings, this.runInteractively); + if (job.hasIngestPipeline()) { + this.jobsById.put(job.getId(), job); + long taskId = nextThreadId.incrementAndGet(); + Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, job)); + ingestJobStarters.put(taskId, task); + } } - - if (!ingestMonitor.isRunning()) { - ingestMonitor.start(); - } - - long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new StartIngestJobsCallable(taskId, dataSource, settings, doStartupErrorsMsgBox)); - startIngestJobsCallables.put(taskId, task); } /** - * Queries whether any ingest jobs are in progress. + * Starts an ingest job that will process a collection of data sources. + * + * @param dataSources The data sources to process. + * @param settings The settings for the ingest job. + * @return The ingest job that was started on success or null on failure. + */ + public synchronized IngestJob startIngestJob(Collection dataSources, IngestJobSettings settings) { + if (this.jobCreationIsEnabled) { + IngestJob job = new IngestJob(dataSources, settings, this.runInteractively); + if (job.hasIngestPipeline()) { + this.jobsById.put(job.getId(), job); + if (this.startIngestJob(job)) { + return job; + } + } + } + return null; + } + + /** + * Starts an ingest job for a collection of data sources. + * + * @param job The ingest job to start. + * @return True if the job was started, false otherwise. + */ + private boolean startIngestJob(IngestJob job) { + boolean success = false; + + if (this.jobCreationIsEnabled) { + if (runInteractively && jobsById.isEmpty()) { // RJCTODO: This is sort of broken + clearIngestMessageBox(); + } + + if (!ingestMonitor.isRunning()) { + ingestMonitor.start(); + } + + List errors = job.start(); + if (errors.isEmpty()) { + this.fireIngestJobStarted(job.getId()); + IngestManager.logger.log(Level.INFO, "Ingest job {0} started", job.getId()); + success = true; + } else { + this.jobsById.remove(job.getId()); + IngestManager.logger.log(Level.INFO, "Ingest job {0} could not be started", job.getId()); + if (this.runInteractively) { + EventQueue.invokeLater(new Runnable() { + + @Override + public void run() { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); + } + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); + } + }); + } + } + } else { + this.jobsById.remove(job.getId()); + } + return success; + } + + synchronized void finishIngestJob(IngestJob job) { + long jobId = job.getId(); + this.jobsById.remove(jobId); + if (!job.isCancelled()) { + IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); + this.fireIngestJobCompleted(jobId); + } else { + IngestManager.logger.log(Level.INFO, "Ingest job {0} cancelled", jobId); + this.fireIngestJobCancelled(jobId); + } + } + + /** + * Queries whether or not any ingest jobs are in progress. * * @return True or false. */ public boolean isIngestRunning() { - return IngestJob.ingestJobsAreRunning(); + return !this.jobsById.isEmpty(); } /** * Cancels all ingest jobs in progress. */ - public void cancelAllIngestJobs() { + public synchronized void cancelAllIngestJobs() { // Stop creating new ingest jobs. - for (Future handle : startIngestJobsCallables.values()) { + for (Future handle : ingestJobStarters.values()) { handle.cancel(true); } // Cancel all the jobs already created. - IngestJob.cancelAllJobs(); + for (IngestJob job : this.jobsById.values()) { + job.cancel(); + } } /** @@ -250,7 +543,7 @@ public class IngestManager { } /** - * Remove an ingest job and ingest module event property change listener. + * Removes an ingest job and ingest module event property change listener. * * @param listener The PropertyChangeListener to unregister. * @deprecated Use removeIngestJobEventListener() and/or @@ -263,85 +556,59 @@ public class IngestManager { } /** - * Starts the ingest monitor and submits task execution tasks (Callable - * objects) to the data source ingest and file ingest thread pools. The task - * execution tasks are simple consumers that will normally run as long as - * the application runs + * Fire an ingest event signifying an ingest job started. + * + * @param ingestJobId The ingest job id. */ - private IngestManager() { - startDataSourceIngestThread(); - - numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); - if ((numberOfFileIngestThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfFileIngestThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { - numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; - UserPreferences.setNumberOfFileIngestThreads(numberOfFileIngestThreads); - } - fileIngestThreadPool = Executors.newFixedThreadPool(numberOfFileIngestThreads); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - startFileIngestThread(); - } + void fireIngestJobStarted(long ingestJobId) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); } /** - * Called by the custom installer for this package once the window system is - * initialized, allowing the ingest manager to get the top component used to - * display ingest messages. + * Fire an ingest event signifying an ingest job finished. + * + * @param ingestJobId The ingest job id. */ - void initIngestMessageInbox() { - ingestMessageBox = IngestMessageTopComponent.findInstance(); + void fireIngestJobCompleted(long ingestJobId) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); } /** - * Submits a ExecuteIngestTasksTask Callable to the data source ingest task - * thread pool. + * Fire an ingest event signifying an ingest job was canceled. + * + * @param ingestJobId The ingest job id. */ - private void startDataSourceIngestThread() { - long threadId = nextThreadId.incrementAndGet(); - dataSourceIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); - ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + void fireIngestJobCancelled(long ingestJobId) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); } /** - * Submits a ExecuteIngestTasksTask Callable to the data source ingest - * thread pool. + * Fire an ingest event signifying the ingest of a file is completed. + * + * @param file The file that is completed. */ - private void startFileIngestThread() { - long threadId = nextThreadId.incrementAndGet(); - fileIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); - ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + void fireFileIngestDone(AbstractFile file) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); } - private void subscribeToCaseEvents() { - Case.addPropertyChangeListener(new PropertyChangeListener() { - @Override - public void propertyChange(PropertyChangeEvent event) { - if (event.getPropertyName().equals(Case.Events.CURRENT_CASE.toString())) { - if (event.getNewValue() != null) { - handleCaseOpened(); - } else { - handleCaseClosed(); - } - } - } - }); + /** + * Fire an event signifying a blackboard post by an ingest module. + * + * @param moduleDataEvent A ModuleDataEvent with the details of the posting. + */ + void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); } - void handleCaseOpened() { - IngestJob.jobCreationEnabled(true); - clearIngestMessageBox(); - } - - void handleCaseClosed() { - IngestJob.jobCreationEnabled(false); - cancelAllIngestJobs(); - clearIngestMessageBox(); - } - - private void clearIngestMessageBox() { - if (ingestMessageBox != null) { - ingestMessageBox.clearMessages(); - } - ingestErrorMessagePosts.set(0); + /** + * Fire an event signifying discovery of additional content by an ingest + * module. + * + * @param moduleDataEvent A ModuleContentEvent with the details of the new + * content. + */ + void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); } /** @@ -386,10 +653,6 @@ public class IngestManager { IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId()); ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); - synchronized (processedFilesSnapshotLock) { - processedFilesSnapshot.incrementProcessedFilesCount(); - } - incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } @@ -437,83 +700,16 @@ public class IngestManager { } /** - * Fire an ingest event signifying an ingest job started. + * Gets snapshots of the state of all running ingest jobs. * - * @param ingestJobId The ingest job id. + * @return A list of ingest job state snapshots. */ - void fireIngestJobStarted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); - } - - /** - * Fire an ingest event signifying an ingest job finished. - * - * @param ingestJobId The ingest job id. - */ - void fireIngestJobCompleted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); - } - - /** - * Fire an ingest event signifying an ingest job was canceled. - * - * @param ingestJobId The ingest job id. - */ - void fireIngestJobCancelled(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); - } - - /** - * Fire an ingest event signifying the ingest of a file is completed. - * - * @param file The file that is completed. - */ - void fireFileIngestDone(AbstractFile file) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); - } - - /** - * Fire an event signifying a blackboard post by an ingest module. - * - * @param moduleDataEvent A ModuleDataEvent with the details of the posting. - */ - void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); - } - - /** - * Fire an event signifying discovery of additional content by an ingest - * module. - * - * @param moduleDataEvent A ModuleContentEvent with the details of the new - * content. - */ - void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); - } - - /** - * Post a message to the ingest messages in box. - * - * @param message The message to be posted. - */ - void postIngestMessage(IngestMessage message) { - if (ingestMessageBox != null) { - if (message.getMessageType() != IngestMessage.MessageType.ERROR && message.getMessageType() != IngestMessage.MessageType.WARNING) { - ingestMessageBox.displayMessage(message); - } else { - long errorPosts = ingestErrorMessagePosts.incrementAndGet(); - if (errorPosts <= MAX_ERROR_MESSAGE_POSTS) { - ingestMessageBox.displayMessage(message); - } else if (errorPosts == MAX_ERROR_MESSAGE_POSTS + 1) { - IngestMessage errorMessageLimitReachedMessage = IngestMessage.createErrorMessage( - NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.title"), - NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.subject"), - NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.msg", MAX_ERROR_MESSAGE_POSTS)); - ingestMessageBox.displayMessage(errorMessageLimitReachedMessage); - } - } + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); + for (IngestJob job : this.jobsById.values()) { + snapShots.addAll(job.getDataSourceIngestJobSnapshots()); } + return snapShots; } /** @@ -531,101 +727,65 @@ public class IngestManager { } /** - * Creates and starts an ingest job, i.e., processing by ingest modules, for - * a data source. + * Creates and starts an ingest job for a collection of data sources. */ - private final class StartIngestJobsCallable implements Callable { + private final class IngestJobStarter implements Callable { private final long threadId; - private final Content dataSource; - private final IngestJobSettings settings; - private final boolean doStartupErrorsMsgBox; + private final IngestJob job; private ProgressHandle progress; - StartIngestJobsCallable(long threadId, Content dataSource, IngestJobSettings settings, boolean doStartupErrorsMsgBox) { + IngestJobStarter(long threadId, IngestJob job) { this.threadId = threadId; - this.dataSource = dataSource; - this.settings = settings; - this.doStartupErrorsMsgBox = doStartupErrorsMsgBox; + this.job = job; } @Override public Void call() { try { if (Thread.currentThread().isInterrupted()) { + jobsById.remove(job.getId()); return null; } - /** - * Set up a progress bar. - */ - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.displayName"); - progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - if (progress != null) { - progress.setDisplayName(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.cancelling", - displayName)); + if (runInteractively) { + final String displayName = NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.displayName"); + this.progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (progress != null) { + progress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.cancelling", displayName)); + } + Future handle = ingestJobStarters.remove(threadId); + handle.cancel(true); + return true; } - Future handle = startIngestJobsCallables.remove(threadId); - handle.cancel(true); - return true; - } - }); - progress.switchToIndeterminate(); - progress.start(); - - /** - * Create and start an ingest job for the data source. - */ - List errors = IngestJob.startJob(dataSource, this.settings); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { - // Report the errors to the user. They have already been logged. - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); - } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); + }); + progress.start(); } - } catch (Exception ex) { - logger.log(Level.SEVERE, "Failed to create ingest job", ex); //NON-NLS - } finally { - progress.finish(); - startIngestJobsCallables.remove(threadId); - } - return null; + startIngestJob(job); + return null; + + } finally { + if (null != progress) { + progress.finish(); + } + ingestJobStarters.remove(threadId); + } } + } /** * A consumer for an ingest task queue. */ - private final class ExecuteIngestTasksRunnable implements Runnable { + private final class IngestTaskExecuter implements Runnable { private final long threadId; private final IngestTaskQueue tasks; - ExecuteIngestTasksRunnable(long threadId, IngestTaskQueue tasks) { + IngestTaskExecuter(long threadId, IngestTaskQueue tasks) { this.threadId = threadId; this.tasks = tasks; } @@ -649,7 +809,7 @@ public class IngestManager { /** * Fires ingest events to ingest manager property change listeners. */ - private static final class FireIngestEventRunnable implements Runnable { + private static final class IngestEventPublisher implements Runnable { private final PropertyChangeSupport publisher; private final IngestJobEvent jobEvent; @@ -657,7 +817,7 @@ public class IngestManager { private final Object oldValue; private final Object newValue; - FireIngestEventRunnable(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) { + IngestEventPublisher(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) { this.publisher = publisher; this.jobEvent = event; this.moduleEvent = null; @@ -665,7 +825,7 @@ public class IngestManager { this.newValue = newValue; } - FireIngestEventRunnable(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) { + IngestEventPublisher(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) { this.publisher = publisher; this.jobEvent = null; this.moduleEvent = event; @@ -748,29 +908,7 @@ public class IngestManager { String getFileName() { return fileName; } - } - static final class ProcessedFilesSnapshot { - - private final Date startTime; - private long processedFilesCount; - - ProcessedFilesSnapshot() { - this.startTime = new Date(); - this.processedFilesCount = 0; - } - - void incrementProcessedFilesCount() { - ++processedFilesCount; - } - - Date getStartTime() { - return startTime; - } - - long getProcessedFilesCount() { - return processedFilesCount; - } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index 17bd6a0dbf..7485b71e4b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -81,4 +81,10 @@ public interface IngestModule { * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ void startUp(IngestJobContext context) throws IngestModuleException; + + /** + * TODO: The next time an API change is legal, add a cancel() method and + * remove the "ingest job is canceled" queries from the IngestJobContext + * class. + */ } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java index 3d4af0b410..90ee2b6078 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java @@ -19,10 +19,14 @@ package org.sleuthkit.autopsy.ingest; import java.awt.BorderLayout; +import java.awt.Container; +import java.awt.Dialog; import java.awt.Dimension; import java.awt.Toolkit; +import java.awt.Window; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; import javax.swing.JDialog; -import javax.swing.JFrame; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; @@ -33,12 +37,39 @@ public final class IngestProgressSnapshotDialog extends JDialog { private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestProgressSnapshotDialog.title.text"); private static final Dimension DIMENSIONS = new Dimension(500, 300); - + /** * Constructs a non-modal instance of the dialog with its own frame. */ public IngestProgressSnapshotDialog() { - super((JFrame) WindowManager.getDefault().getMainWindow(), TITLE, false); + this((Window) WindowManager.getDefault().getMainWindow(), false); + } + + /** + * Constructs an instance of the dialog with its own frame. Could be modal. + * + * @param owner - the owner of this dialog. If this dialog should be modal, the owner gets set to non modal. + * @param shouldBeModal - true if this should be modal, false otherwise. + */ + public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) { + super((Window) owner, TITLE, ModalityType.MODELESS); + if (shouldBeModal && owner instanceof JDialog) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. + final JDialog pseudoOwner = (JDialog) owner; + addWindowListener(new WindowAdapter() { + @Override + public void windowClosed(WindowEvent e) { // Put it back to how it was before we manipulated it. + pseudoOwner.setVisible(false); + pseudoOwner.setModalityType(Dialog.ModalityType.APPLICATION_MODAL); + pseudoOwner.toFront(); + pseudoOwner.setVisible(true); + } + }); + pseudoOwner.setVisible(false); + pseudoOwner.setModalityType(Dialog.ModalityType.MODELESS); + pseudoOwner.toFront(); + pseudoOwner.repaint(); + pseudoOwner.setVisible(true); + } setResizable(true); setLayout(new BorderLayout()); Dimension screenDimension = Toolkit.getDefaultToolkit().getScreenSize(); @@ -49,6 +80,9 @@ public final class IngestProgressSnapshotDialog extends JDialog { add(new IngestProgressSnapshotPanel(this)); pack(); setResizable(false); + if (shouldBeModal) { // if called from a modal dialog, become modal, otherwise don't. + setModal(true); + } setVisible(true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 792a6a4e7f..ce6f0e3c0a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -163,14 +163,14 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final String[] columnNames = {"Job ID", "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; - private List jobSnapshots; + private List jobSnapshots; private IngestJobTableModel() { refresh(); } private void refresh() { - jobSnapshots = IngestJob.getJobSnapshots(); + jobSnapshots = IngestManager.getInstance().getIngestJobSnapshots(); fireTableDataChanged(); } @@ -191,7 +191,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { - IngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); + DataSourceIngestJob.Snapshot snapShot = jobSnapshots.get(rowIndex); Object cellValue; switch (columnIndex) { case 0: @@ -202,7 +202,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { break; case 2: SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); - cellValue = dateFormat.format(new Date(snapShot.getStartTime())); + cellValue = dateFormat.format(new Date(snapShot.getJobStartTime())); break; case 3: cellValue = snapShot.getFilesProcessed(); @@ -429,7 +429,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { }// //GEN-END:initComponents private void closeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_closeButtonActionPerformed - parent.setVisible(false); + parent.dispose(); }//GEN-LAST:event_closeButtonActionPerformed private void refreshButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_refreshButtonActionPerformed diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 7ff6634f64..cfef33be7a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -23,15 +23,15 @@ import org.sleuthkit.datamodel.Content; abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; - private final IngestJob job; + private final DataSourceIngestJob job; private long threadId; - IngestTask(IngestJob job) { + IngestTask(DataSourceIngestJob job) { this.job = job; threadId = NOT_SET; } - IngestJob getIngestJob() { + DataSourceIngestJob getIngestJob() { return job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index d401b1439c..2adfb3ca40 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -148,7 +148,7 @@ final class IngestTasksScheduler { * @throws InterruptedException if the calling thread is blocked due to a * full tasks queue and is interrupted. */ - synchronized void scheduleIngestTasks(IngestJob job) { + synchronized void scheduleIngestTasks(DataSourceIngestJob job) { // Scheduling of both a data source ingest task and file ingest tasks // for a job must be an atomic operation. Otherwise, the data source // task might be completed before the file tasks are scheduled, @@ -163,7 +163,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleDataSourceIngestTask(IngestJob job) { + synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) { DataSourceIngestTask task = new DataSourceIngestTask(job); this.tasksInProgress.add(task); try { @@ -183,7 +183,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleFileIngestTasks(IngestJob job) { + synchronized void scheduleFileIngestTasks(DataSourceIngestJob job) { // Get the top level files for the data source associated with this job // and add them to the root directories priority queue. List topLevelFiles = getTopLevelFiles(job.getDataSource()); @@ -203,7 +203,7 @@ final class IngestTasksScheduler { * @param job The job for which the tasks are to be scheduled. * @param file The file to be associated with the task. */ - synchronized void scheduleFileIngestTask(IngestJob job, AbstractFile file) { + synchronized void scheduleFileIngestTask(DataSourceIngestJob job, AbstractFile file) { FileIngestTask task = new FileIngestTask(job, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { this.tasksInProgress.add(task); @@ -228,7 +228,7 @@ final class IngestTasksScheduler { * @param job The job for which the query is to be performed. * @return True or false. */ - synchronized boolean tasksForJobAreCompleted(IngestJob job) { + synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) { for (IngestTask task : tasksInProgress) { if (task.getIngestJob().getId() == job.getId()) { return false; @@ -245,7 +245,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to to canceled. */ - synchronized void cancelPendingTasksForIngestJob(IngestJob job) { + synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { long jobId = job.getId(); this.removeTasksForJob(this.rootDirectoryTasks, jobId); this.removeTasksForJob(this.directoryTasks, jobId); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index f726a1d525..7e46f9b89e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -198,14 +198,9 @@ public final class RunIngestModulesDialog extends JDialog { IngestJobSettings ingestJobSettings = this.ingestJobSettingsPanel.getSettings(); ingestJobSettings.save(); showWarnings(ingestJobSettings); - if (startIngestJob) { - IngestManager ingestManager = IngestManager.getInstance(); - for (Content dataSource : RunIngestModulesDialog.this.dataSources) { - ingestManager.startIngestJob(dataSource, ingestJobSettings, true); - } + IngestManager.getInstance().queueIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings); } - setVisible(false); dispose(); } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java index f3d1e56019..52439eb6fd 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java @@ -57,7 +57,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Contacts"); + errors.add("Error getting Contacts"); //NON-NLS } try { @@ -67,7 +67,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Call Logs"); + errors.add("Error getting Call Logs"); //NON-NLS } try { @@ -77,7 +77,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Text Messages"); + errors.add("Error getting Text Messages"); //NON-NLS } try { @@ -87,7 +87,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Tango Messages"); + errors.add("Error getting Tango Messages"); //NON-NLS } try { @@ -97,7 +97,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Words with Friends Messages"); + errors.add("Error getting Words with Friends Messages"); //NON-NLS } try { @@ -107,7 +107,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Google Map Locations"); + errors.add("Error getting Google Map Locations"); //NON-NLS } try { @@ -117,14 +117,14 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Browser Locations"); + errors.add("Error getting Browser Locations"); //NON-NLS } try { CacheLocationAnalyzer.findGeoLocations(); progressBar.progress(8); } catch (Exception e) { - errors.add("Error getting Cache Locations"); + errors.add("Error getting Cache Locations"); //NON-NLS } // create the final message for inbox @@ -133,20 +133,20 @@ class AndroidIngestModule implements DataSourceIngestModule { IngestMessage.MessageType msgLevel = IngestMessage.MessageType.INFO; if (errors.isEmpty() == false) { msgLevel = IngestMessage.MessageType.ERROR; - errorMessage.append("Errors were encountered"); + errorMessage.append("Errors were encountered"); //NON-NLS for (String msg : errors) { errorMessage.append("

  • ").append(msg).append("
    • \n"); //NON-NLS } errorMessage.append("\n"); //NON-NLS if (errors.size() == 1) { - errorMsgSubject = "One error was found"; + errorMsgSubject = "One error was found"; //NON-NLS } else { - errorMsgSubject = "errors found: " + errors.size(); + errorMsgSubject = "errors found: " + errors.size(); //NON-NLS } } else { - errorMessage.append("No errors"); - errorMsgSubject = "No errors"; + errorMessage.append("No errors"); //NON-NLS + errorMsgSubject = "No errors"; //NON-NLS } services.postMessage(IngestMessage.createMessage(msgLevel, AndroidModuleFactory.getModuleName(), "Finished Analysis: " + errorMsgSubject, errorMessage.toString())); diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java index 69add3d978..339a7d5fca 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java @@ -43,7 +43,7 @@ class BrowserLocationAnalyzer { public static void findGeoLocations() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List abstractFiles = skCase.findAllFilesWhere("name LIKE 'CachedGeoposition%.db'"); //get exact file names + List abstractFiles = skCase.findAllFilesWhere("name LIKE 'CachedGeoposition%.db'"); //NON-NLS //get exact file names for (AbstractFile abstractFile : abstractFiles) { try { @@ -54,11 +54,11 @@ class BrowserLocationAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findGeoLocationsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Browser Location files", e); + logger.log(Level.SEVERE, "Error parsing Browser Location files", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Browser Location files", e); + logger.log(Level.SEVERE, "Error finding Browser Location files", e); //NON-NLS } } @@ -71,22 +71,22 @@ class BrowserLocationAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error connecting to sql database", e); + logger.log(Level.SEVERE, "Error connecting to sql database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select timestamp, latitude, longitude, accuracy FROM CachedPosition;"); + "Select timestamp, latitude, longitude, accuracy FROM CachedPosition;"); //NON-NLS while (resultSet.next()) { - Long timestamp = Long.valueOf(resultSet.getString("timestamp")) / 1000; - double latitude = Double.valueOf(resultSet.getString("latitude")); - double longitude = Double.valueOf(resultSet.getString("longitude")); + Long timestamp = Long.valueOf(resultSet.getString("timestamp")) / 1000; //NON-NLS + double latitude = Double.valueOf(resultSet.getString("latitude")); //NON-NLS + double longitude = Double.valueOf(resultSet.getString("longitude")); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), moduleName, latitude)); @@ -96,7 +96,7 @@ class BrowserLocationAnalyzer { // bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VALUE.getTypeID(),moduleName, accuracy)); } } catch (Exception e) { - logger.log(Level.SEVERE, "Error Putting artifacts to Blackboard", e); + logger.log(Level.SEVERE, "Error Putting artifacts to Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -105,7 +105,7 @@ class BrowserLocationAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties new file mode 100644 index 0000000000..a091c264ff --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties @@ -0,0 +1 @@ +AndroidModuleFactory.moduleDescription=Android\u30B7\u30B9\u30C6\u30E0\u304A\u3088\u3073\u7B2C\u4E09\u8005\u30A2\u30D7\u30EA\u30C7\u30FC\u30BF\u3092\u62BD\u51FA \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java index 89d5472f25..3e942136f8 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java @@ -43,7 +43,7 @@ class CacheLocationAnalyzer { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List abstractFiles = skCase.findAllFilesWhere("name ='cache.cell' OR name='cache.wifi'"); //get exact file names + List abstractFiles = skCase.findAllFilesWhere("name ='cache.cell' OR name='cache.wifi'"); //NON-NLS //get exact file names for (AbstractFile abstractFile : abstractFiles) { try { @@ -55,11 +55,11 @@ class CacheLocationAnalyzer { findGeoLocationsInFile(jFile, abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing cached Location files", e); + logger.log(Level.SEVERE, "Error parsing cached Location files", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding cached Location files", e); + logger.log(Level.SEVERE, "Error finding cached Location files", e); //NON-NLS } } @@ -124,7 +124,7 @@ class CacheLocationAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Cached GPS locations to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Cached GPS locations to Blackboard", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java index 5f06532860..8c8542f789 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java @@ -47,11 +47,11 @@ class CallLogAnalyzer { /** the where clause(without 'where' of sql select statement to choose call * log dbs, update the list of file names to include more files */ - private static final String fileNameQuery = Stream.of("'logs.db'", "'contacts2.db'", "'contacts.db'") - .collect(Collectors.joining(" OR name = ", "name = ", "")); + private static final String fileNameQuery = Stream.of("'logs.db'", "'contacts2.db'", "'contacts.db'") //NON-NLS + .collect(Collectors.joining(" OR name = ", "name = ", "")); //NON-NLS /** the names of tables that potentially hold call logs in the dbs */ - private static final Iterable tableNames = Arrays.asList("calls", "logs"); + private static final Iterable tableNames = Arrays.asList("calls", "logs"); //NON-NLS public static void findCallLogs() { try { @@ -63,11 +63,11 @@ class CallLogAnalyzer { ContentUtils.writeToFile(abstractFile, file); findCallLogsInDB(file.toString(), abstractFile); } catch (IOException e) { - logger.log(Level.SEVERE, "Error writing temporary call log db to disk", e); + logger.log(Level.SEVERE, "Error writing temporary call log db to disk", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding call logs", e); + logger.log(Level.SEVERE, "Error finding call logs", e); //NON-NLS } } @@ -76,20 +76,20 @@ class CallLogAnalyzer { if (DatabasePath == null || DatabasePath.isEmpty()) { return; } - try (Connection connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + try (Connection connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS Statement statement = connection.createStatement();) { for (String tableName : tableNames) { try (ResultSet resultSet = statement.executeQuery( - "SELECT number,date,duration,type, name FROM " + tableName + " ORDER BY date DESC;");) { - logger.log(Level.INFO, "Reading call log from table {0} in db {1}", new Object[]{tableName, DatabasePath}); + "SELECT number,date,duration,type, name FROM " + tableName + " ORDER BY date DESC;");) { //NON-NLS + logger.log(Level.INFO, "Reading call log from table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NON-NLS while (resultSet.next()) { Long date = resultSet.getLong("date") / 1000; - final CallDirection direction = CallDirection.fromType(resultSet.getInt("type")); + final CallDirection direction = CallDirection.fromType(resultSet.getInt("type")); //NON-NLS String directionString = direction != null ? direction.getDisplayName() : ""; - final String number = resultSet.getString("number"); - final long duration = resultSet.getLong("duration");//duration of call is in seconds - final String name = resultSet.getString("name");// name of person dialed or called. null if unregistered + final String number = resultSet.getString("number"); //NON-NLS + final long duration = resultSet.getLong("duration"); //NON-NLS //duration of call is in seconds + final String name = resultSet.getString("name"); //NON-NLS // name of person dialed or called. null if unregistered try { BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); //create a call log and then add attributes from result set. @@ -104,21 +104,21 @@ class CallLogAnalyzer { bba.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION.getTypeID(), moduleName, directionString)); bba.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error posting call log record to the Blackboard", ex); + logger.log(Level.SEVERE, "Error posting call log record to the Blackboard", ex); //NON-NLS } } } catch (SQLException e) { - logger.log(Level.WARNING, "Could not read table {0} in db {1}", new Object[]{tableName, DatabasePath}); + logger.log(Level.WARNING, "Could not read table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NON-NLS } } } catch (SQLException e) { - logger.log(Level.SEVERE, "Could not parse call log; error connecting to db " + DatabasePath, e); + logger.log(Level.SEVERE, "Could not parse call log; error connecting to db " + DatabasePath, e); //NON-NLS } } private static enum CallDirection { - INCOMING(1, "Incoming"), OUTGOING(2, "Outgoing"), MISSED(3, "Missed"); + INCOMING(1, "Incoming"), OUTGOING(2, "Outgoing"), MISSED(3, "Missed"); //NON-NLS private final int type; diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java index 173d506372..3d4784864f 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java @@ -45,7 +45,7 @@ class ContactAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NON-NLS //get exact file names if (absFiles.isEmpty()) { return; } @@ -55,11 +55,11 @@ class ContactAnalyzer { ContentUtils.writeToFile(AF, jFile); findContactsInDB(jFile.toString(), AF); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts", e); + logger.log(Level.SEVERE, "Error parsing Contacts", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Contacts", e); + logger.log(Level.SEVERE, "Error finding Contacts", e); //NON-NLS } } @@ -78,11 +78,11 @@ class ContactAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } @@ -90,13 +90,13 @@ class ContactAnalyzer { // get display_name, mimetype(email or phone number) and data1 (phonenumber or email address depending on mimetype) //sorted by name, so phonenumber/email would be consecutive for a person if they exist. resultSet = statement.executeQuery( - "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" - + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" - + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " - + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" - + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" - + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" - + "ORDER BY name_raw_contact.display_name ASC;"); + "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NON-NLS + + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NON-NLS + + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NON-NLS + + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NON-NLS + + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NON-NLS + + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NON-NLS + + "ORDER BY name_raw_contact.display_name ASC;"); //NON-NLS BlackboardArtifact bba; bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); @@ -105,15 +105,15 @@ class ContactAnalyzer { String mimetype; // either phone or email String data1; // the phone number or email while (resultSet.next()) { - name = resultSet.getString("display_name"); - data1 = resultSet.getString("data1"); - mimetype = resultSet.getString("mimetype"); + name = resultSet.getString("display_name"); //NON-NLS + data1 = resultSet.getString("data1"); //NON-NLS + mimetype = resultSet.getString("mimetype"); //NON-NLS // System.out.println(resultSet.getString("data1") + resultSet.getString("mimetype") + resultSet.getString("display_name")); //Test code if (name.equals(oldName) == false) { bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } - if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { + if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getTypeID(), moduleName, data1)); } else { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getTypeID(), moduleName, data1)); @@ -122,7 +122,7 @@ class ContactAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -131,7 +131,7 @@ class ContactAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java index 0174b81e1f..88f1aa87ec 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java @@ -44,7 +44,7 @@ class GoogleMapLocationAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='da_destination_history'"); //get exact file name + absFiles = skCase.findAllFilesWhere("name ='da_destination_history'"); //NON-NLS //get exact file name if (absFiles.isEmpty()) { return; } @@ -54,11 +54,11 @@ class GoogleMapLocationAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findGeoLocationsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Google map locations", e); + logger.log(Level.SEVERE, "Error parsing Google map locations", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Google map locations", e); + logger.log(Level.SEVERE, "Error finding Google map locations", e); //NON-NLS } } @@ -71,27 +71,27 @@ class GoogleMapLocationAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select time,dest_lat,dest_lng,dest_title,dest_address,source_lat,source_lng FROM destination_history;"); + "Select time,dest_lat,dest_lng,dest_title,dest_address,source_lat,source_lng FROM destination_history;"); //NON-NLS while (resultSet.next()) { - Long time = Long.valueOf(resultSet.getString("time")) / 1000; - String dest_title = resultSet.getString("dest_title"); - String dest_address = resultSet.getString("dest_address"); + Long time = Long.valueOf(resultSet.getString("time")) / 1000; //NON-NLS + String dest_title = resultSet.getString("dest_title"); //NON-NLS + String dest_address = resultSet.getString("dest_address"); //NON-NLS - double dest_lat = convertGeo(resultSet.getString("dest_lat")); - double dest_lng = convertGeo(resultSet.getString("dest_lng")); - double source_lat = convertGeo(resultSet.getString("source_lat")); - double source_lng = convertGeo(resultSet.getString("source_lng")); + double dest_lat = convertGeo(resultSet.getString("dest_lat")); //NON-NLS + double dest_lng = convertGeo(resultSet.getString("dest_lng")); //NON-NLS + double source_lat = convertGeo(resultSet.getString("source_lat")); //NON-NLS + double source_lng = convertGeo(resultSet.getString("source_lng")); //NON-NLS // bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT);//src @@ -123,7 +123,7 @@ class GoogleMapLocationAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Google map locations to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Google map locations to the Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -132,7 +132,7 @@ class GoogleMapLocationAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing the database", e); + logger.log(Level.SEVERE, "Error closing the database", e); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java index 23e4f07695..cdae0bfcad 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java @@ -45,18 +45,18 @@ class TangoMessageAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='tc.db' "); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='tc.db' "); //NON-NLS //get exact file names for (AbstractFile abstractFile : absFiles) { try { File jFile = new File(Case.getCurrentCase().getTempDirectory(), abstractFile.getName()); ContentUtils.writeToFile(abstractFile, jFile); findTangoMessagesInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Tango messages", e); + logger.log(Level.SEVERE, "Error parsing Tango messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Tango messages", e); + logger.log(Level.SEVERE, "Error finding Tango messages", e); //NON-NLS } } @@ -69,31 +69,31 @@ class TangoMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select conv_id, create_time,direction,payload FROM messages ORDER BY create_time DESC;"); + "Select conv_id, create_time,direction,payload FROM messages ORDER BY create_time DESC;"); //NON-NLS String conv_id; // seems to wrap around the message found in payload after decoding from base-64 String direction; // 1 incoming, 2 outgoing String payload; // seems to be a base64 message wrapped by the conv_id while (resultSet.next()) { - conv_id = resultSet.getString("conv_id"); - Long create_time = Long.valueOf(resultSet.getString("create_time")) / 1000; - if (resultSet.getString("direction").equals("1")) { - direction = "Incoming"; + conv_id = resultSet.getString("conv_id"); //NON-NLS + Long create_time = Long.valueOf(resultSet.getString("create_time")) / 1000; //NON-NLS + if (resultSet.getString("direction").equals("1")) { //NON-NLS + direction = "Incoming"; //NON-NLS } else { - direction = "Outgoing"; + direction = "Outgoing"; //NON-NLS } - payload = resultSet.getString("payload"); + payload = resultSet.getString("payload"); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create a call log and then add attributes from result set. bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), moduleName, create_time)); @@ -104,7 +104,7 @@ class TangoMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Tango messages to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Tango messages to the Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -113,7 +113,7 @@ class TangoMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } @@ -126,7 +126,7 @@ class TangoMessageAnalyzer { String Z = new String(decoded, "UTF-8"); result = Z.split(wrapper)[1]; } catch (Exception e) { - logger.log(Level.SEVERE, "Error decoding a Tango message", e); + logger.log(Level.SEVERE, "Error decoding a Tango message", e); //NON-NLS } return result; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java index 627c64f327..10058dba6b 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java @@ -43,7 +43,7 @@ class TextMessageAnalyzer { public static void findTexts() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //get exact file name + List absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NON-NLS //get exact file name for (AbstractFile abstractFile : absFiles) { try { @@ -51,11 +51,11 @@ class TextMessageAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findTextsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages", e); + logger.log(Level.SEVERE, "Error parsing text messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding text messages", e); + logger.log(Level.SEVERE, "Error finding text messages", e); //NON-NLS } } @@ -68,17 +68,17 @@ class TextMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select address,date,read,type,subject,body FROM sms;"); + "Select address,date,read,type,subject,body FROM sms;"); //NON-NLS String address; // may be phone number, or other addresses @@ -87,15 +87,15 @@ class TextMessageAnalyzer { Integer read; // may be unread = 0, read = 1 String body; //message body while (resultSet.next()) { - address = resultSet.getString("address"); - Long date = Long.valueOf(resultSet.getString("date")) / 1000; + address = resultSet.getString("address"); //NON-NLS + Long date = Long.valueOf(resultSet.getString("date")) / 1000; //NON-NLS - read = resultSet.getInt("read"); - subject = resultSet.getString("subject"); - body = resultSet.getString("body"); + read = resultSet.getInt("read"); //NON-NLS + subject = resultSet.getString("subject"); //NON-NLS + body = resultSet.getString("body"); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create Message artifact and then add attributes from result set. - if (resultSet.getString("type").equals("1")) { + if (resultSet.getString("type").equals("1")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION.getTypeID(), moduleName, "Incoming")); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM.getTypeID(), moduleName, address)); } else { @@ -111,7 +111,7 @@ class TextMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -120,7 +120,7 @@ class TextMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java index 3bda358a24..8977d514be 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java @@ -44,7 +44,7 @@ class WWFMessageAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='WordsFramework' "); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='WordsFramework' "); //NON-NLS //get exact file names for (AbstractFile abstractFile : absFiles) { try { @@ -53,11 +53,11 @@ class WWFMessageAnalyzer { findWWFMessagesInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing WWF messages", e); + logger.log(Level.SEVERE, "Error parsing WWF messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding WWF messages", e); + logger.log(Level.SEVERE, "Error finding WWF messages", e); //NON-NLS } } @@ -74,23 +74,23 @@ class WWFMessageAnalyzer { connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "SELECT message,strftime('%s' ,created_at) as datetime,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); + "SELECT message,strftime('%s' ,created_at) as datetime,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); //NON-NLS String message; // WWF Message String user_id; // the ID of the user who sent the message. String game_id; // ID of the game which the the message was sent. while (resultSet.next()) { - message = resultSet.getString("message"); - Long created_at = resultSet.getLong("datetime"); - user_id = resultSet.getString("user_id"); - game_id = resultSet.getString("game_id"); + message = resultSet.getString("message"); //NON-NLS + Long created_at = resultSet.getLong("datetime"); //NON-NLS + user_id = resultSet.getString("user_id"); //NON-NLS + game_id = resultSet.getString("game_id"); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create a call log and then add attributes from result set. bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), moduleName, created_at)); @@ -100,7 +100,7 @@ class WWFMessageAnalyzer { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MESSAGE_TYPE.getTypeID(), moduleName, "Words With Friends Message")); } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing WWF messages to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing WWF messages to the Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -109,7 +109,7 @@ class WWFMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 1369d3d332..ef33f85fc6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -1,5 +1,4 @@ OpenIDE-Module-Name=FileTypeId -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText=Depending on how many files have known hashes, checking this box will improve the speed of file type identification. FileTypeIdIngestModule.moduleName.text=File Type Identification FileTypeIdIngestModule.moduleDesc.text=Matches file types based on binary signatures. FileTypeIdIngestModule.complete.totalProcTime=Total Processing Time @@ -7,4 +6,38 @@ FileTypeIdIngestModule.complete.totalFiles=Total Files Processed FileTypeIdIngestModule.complete.srvMsg.text=File Type Id Results FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings FileTypeIdModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText=Depending on how many files have known hashes, checking this box will improve the speed of file type identification. +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) +FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text=0x +FileTypeIdGlobalSettingsPanel.deleteTypeButton.text=DeleteType +FileTypeIdGlobalSettingsPanel.offsetTextField.text= +FileTypeIdGlobalSettingsPanel.offsetLabel.text=Offset +FileTypeIdGlobalSettingsPanel.postHitCheckBox.text=Post interesting file hit when found +FileTypeIdGlobalSettingsPanel.signatureTextField.text= +FileTypeIdGlobalSettingsPanel.signatureTypeLabel.text=Signature Type +FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text= +FileTypeIdGlobalSettingsPanel.signatureLabel.text=Signature +FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text=MIME Type +FileTypeIdGlobalSettingsPanel.saveTypeButton.text=Save Type +FileTypeIdGlobalSettingsPanel.signatureComboBox.rawItem=Bytes (Hex) +FileTypeIdGlobalSettingsPanel.signatureComboBox.asciiItem=String (ASCII) +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.message=MIME type is required. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.title=Missing MIME Type +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message=Signature is required. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title=Missing Signature +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message=Offset must be a positive integer. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title=Invalid Offset +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidRawSignatureBytes.message=The signature has one or more invalid hexadecimal digits. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Signature +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.message=Interesting files set name is required if alert is requests. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.title=Missing Interesting Files Set Name +FileTypeIdGlobalSettingsPanel.filesSetNameLabel.text=Files Set Name +FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text= +FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title=Save Failed +FileTypeIdGlobalSettingsPanel.JOptionPane.loadFailed.title=Load Failed +FileTypeIdGlobalSettingsPanel.hintTextArea.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. +FileTypeIdGlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to file type definitions when ingest is running! +UserDefinedFileTypesManager.loadFileTypes.errorMessage=Failed to load existing file type definitions. +UserDefinedFileTypesManager.saveFileTypes.errorMessage=Failed to save file type definitions. +FileTypeIdIngestJobSettingsPanel.skipSmallFilesCheckBox.text=Skip files smaller than {0} bytes +FileTypeIdGlobalSettingsPanel.newTypeButton.text=New Type diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties index a12d0e7cd8..ee53301d77 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties @@ -1,10 +1,10 @@ -OpenIDE-Module-Name=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText=\u65E2\u77E5\u306E\u30CF\u30C3\u30B7\u30E5\u5024\u3092\u6301\u3064\u30D5\u30A1\u30A4\u30EB\u6570\u306B\u3088\u3063\u3066\u306F\u3001\u3053\u306E\u30DC\u30C3\u30AF\u30B9\u3092\u9078\u629E\u3059\u308B\u306E\u306B\u3088\u308A\u3001\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A\u3092\u52A0\u901F\u3057\u307E\u3059\u3002 -FileTypeIdIngestModule.moduleName.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A -FileTypeIdIngestModule.moduleDesc.text=\u30D0\u30A4\u30CA\u30EA\u7F72\u540D\u306B\u57FA\u3065\u3044\u3066\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u3092\u4E00\u81F4\u3059\u308B\u3002 -FileTypeIdIngestModule.complete.totalProcTime=\u5408\u8A08\u51E6\u7406\u6642\u9593 -FileTypeIdIngestModule.complete.totalFiles=\u5408\u8A08\u51E6\u7406\u30D5\u30A1\u30A4\u30EB\u6570 -FileTypeIdIngestModule.complete.srvMsg.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u7279\u5B9A\u306E\u7D50\u679C -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\uFF08NSRL\uFF09\u3092\u30B9\u30AD\u30C3\u30D7 -FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=\u8A2D\u5B9A\u3092\u884C\u3046\u70BA\u306E\u60F3\u5B9A\u3055\u308C\u308B\u5F15\u6570\u306Finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 -FileTypeIdModuleFactory.createFileIngestModule.exception.msg=\u8A2D\u5B9A\u3092\u884C\u3046\u70BA\u306E\u60F3\u5B9A\u3055\u308C\u308B\u5F15\u6570\u306Finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 \ No newline at end of file +OpenIDE-Module-Name=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a +FileTypeIdIngestModule.moduleName.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a +FileTypeIdIngestModule.moduleDesc.text=\u30d0\u30a4\u30ca\u30ea\u7f72\u540d\u306b\u57fa\u3065\u3044\u3066\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u3092\u4e00\u81f4\u3059\u308b\u3002 +FileTypeIdIngestModule.complete.totalProcTime=\u5408\u8a08\u51e6\u7406\u6642\u9593 +FileTypeIdIngestModule.complete.totalFiles=\u5408\u8a08\u51e6\u7406\u30d5\u30a1\u30a4\u30eb\u6570 +FileTypeIdIngestModule.complete.srvMsg.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u7279\u5b9a\u306e\u7d50\u679c +FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=\u8a2d\u5b9a\u3092\u884c\u3046\u70ba\u306e\u60f3\u5b9a\u3055\u308c\u308b\u5f15\u6570\u306finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 +FileTypeIdModuleFactory.createFileIngestModule.exception.msg=\u8a2d\u5b9a\u3092\u884c\u3046\u70ba\u306e\u60f3\u5b9a\u3055\u308c\u308b\u5f15\u6570\u306finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText=\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u6301\u3064\u30d5\u30a1\u30a4\u30eb\u6570\u306b\u3088\u3063\u3066\u306f\u3001\u3053\u306e\u30dc\u30c3\u30af\u30b9\u3092\u9078\u629e\u3059\u308b\u306e\u306b\u3088\u308a\u3001\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a\u3092\u52a0\u901f\u3057\u307e\u3059\u3002 +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text=\u65e2\u77e5\u30d5\u30a1\u30a4\u30eb\uff08NSRL\uff09\u3092\u30b9\u30ad\u30c3\u30d7 diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java new file mode 100644 index 0000000000..e3903ffc30 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -0,0 +1,193 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.util.Arrays; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Represents a file type characterized by a file signature. + *

    + * Thread-safe (immutable). + */ +class FileType { + + private final String mimeType; + private final Signature signature; + private final String interestingFilesSetName; + private final boolean alert; + + /** + * Creates a representation of a file type characterized by a file + * signature. + * + * @param mimeType The mime type to associate with this file type. + * @param signature The signature that characterizes this file type. + * @param filesSetName The name of an interesting files set that includes + * files of this type, may be the empty string. + * @param alert Whether the user wishes to be alerted when a file matching + * this type is encountered. + */ + FileType(String mimeType, final Signature signature, String filesSetName, boolean alert) { + this.mimeType = mimeType; + this.signature = new Signature(signature.getSignatureBytes(), signature.getOffset(), signature.getType()); + this.interestingFilesSetName = filesSetName; + this.alert = alert; + } + + /** + * Gets the MIME type associated with this file type. + * + * @return The MIME type. + */ + String getMimeType() { + return mimeType; + } + + /** + * Gets the signature associated with this file type. + * + * @return The signature. + */ + Signature getSignature() { + return new Signature(signature.getSignatureBytes(), signature.getOffset(), signature.getType()); + } + + /** + * Determines whether or not a file is an instance of this file type. + * + * @param file The file to test. + * @return True or false. + */ + boolean matches(final AbstractFile file) { + return signature.containedIn(file); + } + + /** + * Indicates whether or not an alert is desired if a file of this type is + * encountered. + * + * @return True or false. + */ + boolean alertOnMatch() { + return alert; + } + + /** + * Gets the name of an interesting files set that includes files of this + * type. + * + * @return The interesting files set name, possibly empty. + */ + String getFilesSetName() { + return interestingFilesSetName; + } + + /** + * A file signature consisting of a sequence of bytes at a specific offset + * within a file. + *

    + * Thread-safe (immutable). + */ + static class Signature { + + private static final Logger logger = Logger.getLogger(Signature.class.getName()); + + /** + * The way the signature byte sequence should be interpreted. + */ + enum Type { + + RAW, ASCII + }; + + private final byte[] signatureBytes; + private final long offset; + private final Type type; + + /** + * Creates a file signature consisting of a sequence of bytes at a + * specific offset within a file. + * + * @param signatureBytes The signature bytes. + * @param offset The offset of the signature bytes. + * @param type The interpretation of the signature bytes (e.g., raw + * bytes, an ASCII string). + */ + Signature(final byte[] signatureBytes, long offset, Type type) { + this.signatureBytes = Arrays.copyOf(signatureBytes, signatureBytes.length); + this.offset = offset; + this.type = type; + } + + /** + * Gets the byte sequence of the signature. + * + * @return The byte sequence as an array of bytes. + */ + byte[] getSignatureBytes() { + return Arrays.copyOf(signatureBytes, signatureBytes.length); + } + + /** + * Gets the offset of the signature. + * + * @return The offset. + */ + long getOffset() { + return offset; + } + + /** + * Gets the interpretation of the byte sequence for the signature. + * + * @return The signature type. + */ + Type getType() { + return type; + } + + /** + * Determines whether or not the signature is contained within a given + * file. + * + * @param file The file to test + * @return True or false. + */ + boolean containedIn(final AbstractFile file) { + try { + byte[] buffer = new byte[signatureBytes.length]; + int bytesRead = file.read(buffer, offset, signatureBytes.length); + return ((bytesRead == signatureBytes.length) && (Arrays.equals(buffer, signatureBytes))); + } catch (TskCoreException ex) { + /** + * This exception is caught rather than propagated because files + * in images are not always consistent with their file system + * meta data making for read errors. + */ + Signature.logger.log(Level.WARNING, "Error reading from file with objId = " + file.getId(), ex); + return false; + } + } + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form new file mode 100644 index 0000000000..c168205dad --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -0,0 +1,346 @@ + + +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java new file mode 100644 index 0000000000..d0847fc63f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -0,0 +1,672 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.awt.EventQueue; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; +import javax.swing.DefaultComboBoxModel; +import javax.swing.DefaultListModel; +import javax.swing.JOptionPane; +import javax.swing.event.DocumentEvent; +import javax.swing.event.DocumentListener; +import javax.swing.event.ListSelectionEvent; +import javax.swing.event.ListSelectionListener; +import javax.xml.bind.DatatypeConverter; +import org.openide.util.Exceptions; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.corecomponents.OptionsPanel; +import org.sleuthkit.autopsy.ingest.IngestManager; +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; +import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; +import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException; + +/** + * A panel to allow a user to make custom file type definitions. In addition to + * being an ingest module global settings panel, an instance of this class also + * appears in the NetBeans options dialog as an options panel. + */ +final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { + + private static final String RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM = NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureComboBox.rawItem"); + private static final String ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM = NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureComboBox.asciiItem"); + + /** + * The list model for the file types list component of this panel is the set + * of MIME types associated with the user-defined file types. A mapping of + * the MIME types to file type objects lies behind the list model. This map + * is obtained from the user-defined types manager. + */ + private DefaultListModel typesListModel; + private Map fileTypes; + + /** + * This panel implements a property change listener that listens to ingest + * job events so it can disable the buttons on the panel if ingest is + * running. This is done to prevent changes to user-defined types while the + * type definitions are in use. + */ + // TODO: Disabling during ingest would not be necessary if the file ingest + // modules obtained and shared a per data source ingest job snapshot of the + // file type definitions. + IngestJobEventPropertyChangeListener ingestJobEventsListener; + + /** + * Creates a panel to allow a user to make custom file type definitions. + */ + FileTypeIdGlobalSettingsPanel() { + initComponents(); + customizeComponents(); + addIngestJobEventsListener(); + } + + /** + * Does child component initialization in addition to that done by the + * Matisse generated code. + */ + private void customizeComponents() { + setFileTypesListModel(); + setSignatureTypeComboBoxModel(); + clearTypeDetailsComponents(); + addTypeListSelectionListener(); + addTextFieldListeners(); + } + + /** + * Sets the list model for the list of file types. + */ + private void setFileTypesListModel() { + typesListModel = new DefaultListModel<>(); + typesList.setModel(typesListModel); + } + + /** + * Sets the model for the signature type combo box. + */ + private void setSignatureTypeComboBoxModel() { + DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + signatureTypeComboBox.setModel(sigTypeComboBoxModel); + signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + } + + /** + * Adds a listener to the types list component so that the components in the + * file type details section of the panel can be populated and cleared based + * on the selection in the list. + */ + private void addTypeListSelectionListener() { + typesList.addListSelectionListener(new ListSelectionListener() { + @Override + public void valueChanged(ListSelectionEvent e) { + if (e.getValueIsAdjusting() == false) { + if (typesList.getSelectedIndex() == -1) { + clearTypeDetailsComponents(); + } else { + populateTypeDetailsComponents(); + } + } + } + }); + } + + /** + * Adds listeners to the text fields that enable and disable the buttons on + * the panel. + */ + private void addTextFieldListeners() { + DocumentListener listener = new DocumentListener() { + @Override + public void changedUpdate(DocumentEvent e) { + enableButtons(); + } + + @Override + public void removeUpdate(DocumentEvent e) { + enableButtons(); + } + + @Override + public void insertUpdate(DocumentEvent e) { + enableButtons(); + } + }; + + mimeTypeTextField.getDocument().addDocumentListener(listener); + offsetTextField.getDocument().addDocumentListener(listener); + signatureTextField.getDocument().addDocumentListener(listener); + filesSetNameTextField.getDocument().addDocumentListener(listener); + } + + /** + * Add a property change listener that listens to ingest job events to + * disable the buttons on the panel if ingest is running. This is done to + * prevent changes to user-defined types while the type definitions are in + * use. + */ + // TODO: Disabling during ingest would not be necessary if the file ingest + // modules obtained and shared a per data source ingest job snapshot of the + // file type definitions. + private void addIngestJobEventsListener() { + ingestJobEventsListener = new IngestJobEventPropertyChangeListener(); + IngestManager.getInstance().addIngestJobEventListener(ingestJobEventsListener); + } + + /** + * A property change listener that listens to ingest job events. + */ + private class IngestJobEventPropertyChangeListener implements PropertyChangeListener { + + @Override + public void propertyChange(PropertyChangeEvent evt) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + enableButtons(); + } + }); + } + } + + /** + * Enables or disables the panel buttons based on the state of the panel and + * the application. + */ + private void enableButtons() { + boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning(); + newTypeButton.setEnabled(!ingestIsRunning); + + boolean fileTypeIsSelected = typesList.getSelectedIndex() != -1; + deleteTypeButton.setEnabled(!ingestIsRunning && fileTypeIsSelected); + + boolean requiredFieldsPopulated + = !mimeTypeTextField.getText().isEmpty() + && !offsetTextField.getText().isEmpty() + && !signatureTextField.getText().isEmpty() + && (postHitCheckBox.isSelected() ? !filesSetNameTextField.getText().isEmpty() : true); + saveTypeButton.setEnabled(!ingestIsRunning && requiredFieldsPopulated); + + ingestRunningWarningLabel.setVisible(ingestIsRunning); + } + + /** + * @inheritDoc + */ + @Override + public void load() { + try { + fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); + updateFileTypesListModel(); + if (!typesListModel.isEmpty()) { + typesList.setSelectedIndex(0); + } + } catch (UserDefinedFileTypesException ex) { + JOptionPane.showMessageDialog(null, + ex.getLocalizedMessage(), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.loadFailed.title"), + JOptionPane.ERROR_MESSAGE); + fileTypes = Collections.emptyMap(); + } + enableButtons(); + } + + /** + * Sets the list model for the file types list component. + */ + private void updateFileTypesListModel() { + ArrayList mimeTypes = new ArrayList<>(fileTypes.keySet()); + Collections.sort(mimeTypes); + typesListModel.clear(); + for (String mimeType : mimeTypes) { + typesListModel.addElement(mimeType); + } + } + + /** + * Populates all of the components in the file type details portion of the + * panel based on the current selection in the file types list. + */ + private void populateTypeDetailsComponents() { + String mimeType = typesList.getSelectedValue(); + FileType fileType = fileTypes.get(mimeType); + if (null != fileType) { + mimeTypeTextField.setText(fileType.getMimeType()); + Signature signature = fileType.getSignature(); + FileType.Signature.Type sigType = signature.getType(); + signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + String signatureBytes; + if (Signature.Type.RAW == signature.getType()) { + signatureBytes = DatatypeConverter.printHexBinary(signature.getSignatureBytes()); + } else { + try { + signatureBytes = new String(signature.getSignatureBytes(), "UTF-8"); + } catch (UnsupportedEncodingException ex) { + JOptionPane.showMessageDialog(null, + ex.getLocalizedMessage(), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title"), + JOptionPane.ERROR_MESSAGE); + signatureBytes = ""; + } + } + signatureTextField.setText(signatureBytes); + offsetTextField.setText(Long.toString(signature.getOffset())); + postHitCheckBox.setSelected(fileType.alertOnMatch()); + filesSetNameTextField.setEnabled(postHitCheckBox.isSelected()); + filesSetNameTextField.setText(fileType.getFilesSetName()); + } + enableButtons(); + } + + /** + * Clears all of the components in the individual type details portion of + * the panel. + */ + private void clearTypeDetailsComponents() { + typesList.clearSelection(); + mimeTypeTextField.setText(""); //NON-NLS + signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + hexPrefixLabel.setVisible(true); + signatureTextField.setText(""); //NON-NLS + offsetTextField.setText(""); //NON-NLS + postHitCheckBox.setSelected(false); + filesSetNameTextField.setText(""); //NON-NLS + filesSetNameTextField.setEnabled(false); + enableButtons(); + } + + /** + * @inheritDoc + */ + @Override + public void store() { + try { + UserDefinedFileTypesManager.getInstance().setUserDefinedFileTypes(fileTypes); + } catch (UserDefinedFileTypesManager.UserDefinedFileTypesException ex) { + JOptionPane.showMessageDialog(null, + ex.getLocalizedMessage(), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title"), + JOptionPane.ERROR_MESSAGE); + } + } + + /** + * @inheritDoc + */ + @Override + public void saveSettings() { + store(); + } + + /** + * @inheritDoc + */ + @Override + @SuppressWarnings("FinalizeDeclaration") + protected void finalize() throws Throwable { + IngestManager.getInstance().removeIngestJobEventListener(ingestJobEventsListener); + super.finalize(); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + typesScrollPane = new javax.swing.JScrollPane(); + typesList = new javax.swing.JList(); + separator = new javax.swing.JSeparator(); + mimeTypeLabel = new javax.swing.JLabel(); + mimeTypeTextField = new javax.swing.JTextField(); + signatureTypeLabel = new javax.swing.JLabel(); + signatureTextField = new javax.swing.JTextField(); + offsetLabel = new javax.swing.JLabel(); + offsetTextField = new javax.swing.JTextField(); + newTypeButton = new javax.swing.JButton(); + deleteTypeButton = new javax.swing.JButton(); + saveTypeButton = new javax.swing.JButton(); + hexPrefixLabel = new javax.swing.JLabel(); + signatureTypeComboBox = new javax.swing.JComboBox(); + signatureLabel = new javax.swing.JLabel(); + hintScrollPane = new javax.swing.JScrollPane(); + hintTextArea = new javax.swing.JTextArea(); + postHitCheckBox = new javax.swing.JCheckBox(); + filesSetNameLabel = new javax.swing.JLabel(); + filesSetNameTextField = new javax.swing.JTextField(); + ingestRunningWarningLabel = new javax.swing.JLabel(); + + setMaximumSize(new java.awt.Dimension(500, 300)); + setPreferredSize(new java.awt.Dimension(500, 300)); + + typesList.setMinimumSize(new java.awt.Dimension(200, 0)); + typesScrollPane.setViewportView(typesList); + + separator.setOrientation(javax.swing.SwingConstants.VERTICAL); + + org.openide.awt.Mnemonics.setLocalizedText(mimeTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text")); // NOI18N + + mimeTypeTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(signatureTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureTypeLabel.text")); // NOI18N + + signatureTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureTextField.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(offsetLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.offsetLabel.text")); // NOI18N + + offsetTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.offsetTextField.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(newTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.newTypeButton.text")); // NOI18N + newTypeButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + newTypeButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(deleteTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.deleteTypeButton.text")); // NOI18N + deleteTypeButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + deleteTypeButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(saveTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.saveTypeButton.text")); // NOI18N + saveTypeButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + saveTypeButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(hexPrefixLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text")); // NOI18N + + signatureTypeComboBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + signatureTypeComboBoxActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(signatureLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureLabel.text")); // NOI18N + + hintTextArea.setEditable(false); + hintTextArea.setColumns(20); + hintTextArea.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N + hintTextArea.setLineWrap(true); + hintTextArea.setRows(5); + hintTextArea.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.hintTextArea.text")); // NOI18N + hintTextArea.setWrapStyleWord(true); + hintTextArea.setPreferredSize(new java.awt.Dimension(164, 70)); + hintScrollPane.setViewportView(hintTextArea); + + org.openide.awt.Mnemonics.setLocalizedText(postHitCheckBox, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.postHitCheckBox.text")); // NOI18N + postHitCheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + postHitCheckBoxActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(filesSetNameLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.filesSetNameLabel.text")); // NOI18N + + filesSetNameTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text")); // NOI18N + + ingestRunningWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/filetypeid/warning16.png"))); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(ingestRunningWarningLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.ingestRunningWarningLabel.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(ingestRunningWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGap(30, 30, 30)) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE) + .addGroup(layout.createSequentialGroup() + .addComponent(deleteTypeButton) + .addGap(0, 97, Short.MAX_VALUE))) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) + .addGroup(layout.createSequentialGroup() + .addComponent(offsetLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 71, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) + .addComponent(signatureLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(signatureTypeLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGap(2, 2, 2) + .addComponent(mimeTypeLabel))) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 176, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 176, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addComponent(hexPrefixLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 160, javax.swing.GroupLayout.PREFERRED_SIZE))))) + .addComponent(hintScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(postHitCheckBox) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) + .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 179, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(newTypeButton, javax.swing.GroupLayout.PREFERRED_SIZE, 71, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(saveTypeButton)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addComponent(filesSetNameLabel) + .addGap(188, 188, 188))))) + .addGap(29, 29, 29)))) + ); + + layout.linkSize(javax.swing.SwingConstants.HORIZONTAL, new java.awt.Component[] {deleteTypeButton, newTypeButton, saveTypeButton}); + + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(hintScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(mimeTypeLabel) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(signatureTypeLabel) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(hexPrefixLabel) + .addComponent(signatureLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(offsetLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(postHitCheckBox) + .addGap(1, 1, 1) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(filesSetNameLabel) + .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(newTypeButton) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(deleteTypeButton) + .addComponent(saveTypeButton)))) + .addGroup(layout.createSequentialGroup() + .addGap(11, 11, 11) + .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 257, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(ingestRunningWarningLabel) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + ); + + layout.linkSize(javax.swing.SwingConstants.VERTICAL, new java.awt.Component[] {deleteTypeButton, newTypeButton, saveTypeButton}); + + }// //GEN-END:initComponents + + private void newTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_newTypeButtonActionPerformed + clearTypeDetailsComponents(); + }//GEN-LAST:event_newTypeButtonActionPerformed + + private void deleteTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteTypeButtonActionPerformed + String typeName = typesList.getSelectedValue(); + fileTypes.remove(typeName); + updateFileTypesListModel(); + if (!typesListModel.isEmpty()) { + typesList.setSelectedIndex(0); + } + }//GEN-LAST:event_deleteTypeButtonActionPerformed + + private void saveTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_saveTypeButtonActionPerformed + /** + * Get the MIME type. + */ + String typeName = mimeTypeTextField.getText(); + if (typeName.isEmpty()) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.title"), + JOptionPane.ERROR_MESSAGE); + return; + } + + /** + * Get the signature type. + */ + FileType.Signature.Type sigType = signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; + + /** + * Get the signature bytes. + */ + String sigString = signatureTextField.getText(); + if (sigString.isEmpty()) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title"), + JOptionPane.ERROR_MESSAGE); + return; + } + byte[] signatureBytes; + if (FileType.Signature.Type.RAW == sigType) { + try { + signatureBytes = DatatypeConverter.parseHexBinary(sigString); + } catch (IllegalArgumentException ex) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidRawSignatureBytes.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title"), + JOptionPane.ERROR_MESSAGE); + return; + } + } else { + signatureBytes = sigString.getBytes(Charset.forName("UTF-8")); + } + + /** + * Get the offset. + */ + long offset; + try { + offset = Long.parseUnsignedLong(offsetTextField.getText()); + } catch (NumberFormatException ex) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), + JOptionPane.ERROR_MESSAGE); + return; + } + + /** + * Get the interesting files set details. + */ + String filesSetName = filesSetNameTextField.getText(); + if (postHitCheckBox.isSelected() && filesSetName.isEmpty()) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.title"), + JOptionPane.ERROR_MESSAGE); + return; + } + + /** + * Put it all together and reset the file types list component. + */ + FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); + FileType fileType = new FileType(typeName, signature, filesSetName, postHitCheckBox.isSelected()); + fileTypes.put(typeName, fileType); + updateFileTypesListModel(); + typesList.setSelectedValue(fileType.getMimeType(), true); + }//GEN-LAST:event_saveTypeButtonActionPerformed + + private void postHitCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_postHitCheckBoxActionPerformed + filesSetNameTextField.setEnabled(postHitCheckBox.isSelected()); + enableButtons(); + }//GEN-LAST:event_postHitCheckBoxActionPerformed + + private void signatureTypeComboBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_signatureTypeComboBoxActionPerformed + hexPrefixLabel.setVisible(signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + }//GEN-LAST:event_signatureTypeComboBoxActionPerformed + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JButton deleteTypeButton; + private javax.swing.JLabel filesSetNameLabel; + private javax.swing.JTextField filesSetNameTextField; + private javax.swing.JLabel hexPrefixLabel; + private javax.swing.JScrollPane hintScrollPane; + private javax.swing.JTextArea hintTextArea; + private javax.swing.JLabel ingestRunningWarningLabel; + private javax.swing.JLabel mimeTypeLabel; + private javax.swing.JTextField mimeTypeTextField; + private javax.swing.JButton newTypeButton; + private javax.swing.JLabel offsetLabel; + private javax.swing.JTextField offsetTextField; + private javax.swing.JCheckBox postHitCheckBox; + private javax.swing.JButton saveTypeButton; + private javax.swing.JSeparator separator; + private javax.swing.JLabel signatureLabel; + private javax.swing.JTextField signatureTextField; + private javax.swing.JComboBox signatureTypeComboBox; + private javax.swing.JLabel signatureTypeLabel; + private javax.swing.JList typesList; + private javax.swing.JScrollPane typesScrollPane; + // End of variables declaration//GEN-END:variables + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form similarity index 61% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form index 60dfbc7cd3..f03a25070c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form @@ -18,7 +18,10 @@ - + + + + @@ -28,7 +31,9 @@ - + + + @@ -38,15 +43,25 @@ - + - + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java similarity index 61% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java index 3839d2c711..87dd8dd53e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; @@ -25,25 +26,34 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; * UI component used to set ingest job options for file type identifier ingest * modules. */ -final class FileTypeIdModuleSettingsPanel extends IngestModuleIngestJobSettingsPanel { +class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPanel { private final FileTypeIdModuleSettings settings; - public FileTypeIdModuleSettingsPanel(FileTypeIdModuleSettings settings) { + FileTypeIdIngestJobSettingsPanel(FileTypeIdModuleSettings settings) { this.settings = settings; initComponents(); customizeComponents(); } - private void customizeComponents() { - skipKnownCheckBox.setSelected(settings.skipKnownFiles()); - } - + /** + * @inheritDoc + */ @Override public IngestModuleIngestJobSettings getSettings() { return settings; } + /** + * Does child component initialization in addition to that done by the + * Matisse generated code. + */ + private void customizeComponents() { + skipKnownCheckBox.setSelected(settings.skipKnownFiles()); + skipSmallFilesCheckBox.setSelected(settings.skipSmallFiles()); + skipSmallFilesCheckBox.setText(NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipSmallFilesCheckBox.text", settings.minFileSizeInBytes())); + } + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -54,23 +64,33 @@ final class FileTypeIdModuleSettingsPanel extends IngestModuleIngestJobSettingsP private void initComponents() { skipKnownCheckBox = new javax.swing.JCheckBox(); + skipSmallFilesCheckBox = new javax.swing.JCheckBox(); skipKnownCheckBox.setSelected(true); - skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdModuleSettingsPanel.class, "FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text")); // NOI18N - skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdModuleSettingsPanel.class, "FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText")); // NOI18N + skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text")); // NOI18N + skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText")); // NOI18N skipKnownCheckBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { skipKnownCheckBoxActionPerformed(evt); } }); + skipSmallFilesCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipSmallFilesCheckBox.text")); // NOI18N + skipSmallFilesCheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + skipSmallFilesCheckBoxActionPerformed(evt); + } + }); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) - .addComponent(skipKnownCheckBox) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(skipSmallFilesCheckBox) + .addComponent(skipKnownCheckBox)) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); layout.setVerticalGroup( @@ -78,14 +98,22 @@ final class FileTypeIdModuleSettingsPanel extends IngestModuleIngestJobSettingsP .addGroup(layout.createSequentialGroup() .addGap(11, 11, 11) .addComponent(skipKnownCheckBox) - .addContainerGap(47, Short.MAX_VALUE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(skipSmallFilesCheckBox) + .addContainerGap(60, Short.MAX_VALUE)) ); }// //GEN-END:initComponents private void skipKnownCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipKnownCheckBoxActionPerformed settings.setSkipKnownFiles(skipKnownCheckBox.isSelected()); }//GEN-LAST:event_skipKnownCheckBoxActionPerformed + + private void skipSmallFilesCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipSmallFilesCheckBoxActionPerformed + settings.setSkipSmallFiles(skipSmallFilesCheckBox.isSelected()); + }//GEN-LAST:event_skipSmallFilesCheckBoxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JCheckBox skipKnownCheckBox; + private javax.swing.JCheckBox skipSmallFilesCheckBox; // End of variables declaration//GEN-END:variables } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 2abb69974a..9713efd899 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -20,127 +20,38 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.util.HashMap; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; /** * Detects the type of a file based on signature (magic) values. Posts results * to the blackboard. */ +// TODO: This class does not need to be public. public class FileTypeIdIngestModule implements FileIngestModule { private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); - private static final long MIN_FILE_SIZE = 512; private final FileTypeIdModuleSettings settings; - private long jobId; - + private long jobId; private static final HashMap totalsForIngestJobs = new HashMap<>(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); - private TikaFileTypeDetector tikaDetector = new TikaFileTypeDetector(); + private final UserDefinedFileTypeDetector userDefinedFileTypeIdentifier; + private final TikaFileTypeDetector tikaDetector = new TikaFileTypeDetector(); - private static class IngestJobTotals { - long matchTime = 0; - long numFiles = 0; - } - - /** - * Update the match time total and increment num of files for this job - * @param ingestJobId - * @param matchTimeInc amount of time to add - */ - private static synchronized void addToTotals(long ingestJobId, long matchTimeInc) { - IngestJobTotals ingestJobTotals = totalsForIngestJobs.get(ingestJobId); - if (ingestJobTotals == null) { - ingestJobTotals = new IngestJobTotals(); - totalsForIngestJobs.put(ingestJobId, ingestJobTotals); - } - - ingestJobTotals.matchTime += matchTimeInc; - ingestJobTotals.numFiles++; - totalsForIngestJobs.put(ingestJobId, ingestJobTotals); - } - - FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { - this.settings = settings; - } - - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - jobId = context.getJobId(); - refCounter.incrementAndGet(jobId); - } - - @Override - public ProcessResult process(AbstractFile abstractFile) { - // skip non-files - if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) - || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { - - return ProcessResult.OK; - } - - if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) { - return ProcessResult.OK; - } - - if (abstractFile.getSize() < MIN_FILE_SIZE) { - return ProcessResult.OK; - } - - try { - long startTime = System.currentTimeMillis(); - tikaDetector.detectAndSave(abstractFile); - addToTotals(jobId, (System.currentTimeMillis() - startTime)); //add match time - return ProcessResult.OK; - } catch (TskException ex) { - logger.log(Level.WARNING, "Error matching file signature", ex); //NON-NLS - return ProcessResult.ERROR; - } catch (Exception e) { - logger.log(Level.WARNING, "Error matching file signature", e); //NON-NLS - return ProcessResult.ERROR; - } - } - - @Override - public void shutDown() { - // We only need to post the summary msg from the last module per job - if (refCounter.decrementAndGet(jobId) == 0) { - IngestJobTotals jobTotals; - synchronized(this) { - jobTotals = totalsForIngestJobs.remove(jobId); - } - if (jobTotals != null) { - StringBuilder detailsSb = new StringBuilder(); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - detailsSb.append("\n"); //NON-NLS - detailsSb.append("\n"); //NON-NLS - detailsSb.append("
    ").append(FileTypeIdModuleFactory.getModuleName()).append("
    ") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) - .append("").append(jobTotals.matchTime).append("
    ") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) - .append("").append(jobTotals.numFiles).append("
    "); //NON-NLS - IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "FileTypeIdIngestModule.complete.srvMsg.text"), - detailsSb.toString())); - } - } - } - /** * Validate if a given mime type is in the detector's registry. * @@ -157,4 +68,157 @@ public class FileTypeIdIngestModule implements FileIngestModule { TikaFileTypeDetector detector = new TikaFileTypeDetector(); return detector.isMimeTypeDetectable(mimeType); } -} \ No newline at end of file + + /** + * Creates an ingest module that detects the type of a file based on + * signature (magic) values. Posts results to the blackboard. + * + * @param settings The ingest module settings. + */ + FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { + this.settings = settings; + userDefinedFileTypeIdentifier = new UserDefinedFileTypeDetector(); + try { + userDefinedFileTypeIdentifier.loadFileTypes(); + } catch (UserDefinedFileTypesManager.UserDefinedFileTypesException ex) { + logger.log(Level.SEVERE, "Failed to load file types", ex); + MessageNotifyUtil.Notify.error(FileTypeIdModuleFactory.getModuleName(), ex.getMessage()); + } + } + + /** + * @inheritDoc + */ + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + refCounter.incrementAndGet(jobId); + } + + /** + * @inheritDoc + */ + @Override + public ProcessResult process(AbstractFile file) { + + String name = file.getName(); + + /** + * Skip unallocated space and unused blocks files. + */ + if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { + + return ProcessResult.OK; + } + + /** + * Skip known files if configured to do so. + */ + if (settings.skipKnownFiles() && (file.getKnown() == FileKnown.KNOWN)) { + return ProcessResult.OK; + } + + /** + * Filter out very small files to minimize false positives. + */ + if (settings.skipSmallFiles() && file.getSize() < settings.minFileSizeInBytes()) { + return ProcessResult.OK; + } + + try { + long startTime = System.currentTimeMillis(); + FileType fileType = this.userDefinedFileTypeIdentifier.identify(file); + if (null != fileType) { + String moduleName = FileTypeIdModuleFactory.getModuleName(); + BlackboardArtifact getInfoArtifact = file.getGenInfoArtifact(); + BlackboardAttribute typeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID(), moduleName, fileType.getMimeType()); + getInfoArtifact.addAttribute(typeAttr); + + if (fileType.alertOnMatch()) { + BlackboardArtifact artifact = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); + BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), moduleName, fileType.getFilesSetName()); + artifact.addAttribute(setNameAttribute); + + /** + * Use the MIME type as the category, i.e., the rule that + * determined this file belongs to the interesting files + * set. + */ + BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY.getTypeID(), moduleName, fileType.getMimeType()); + artifact.addAttribute(ruleNameAttribute); + } + + } else { + tikaDetector.detectAndSave(file); + } + addToTotals(jobId, (System.currentTimeMillis() - startTime)); + return ProcessResult.OK; + } catch (TskException ex) { + logger.log(Level.WARNING, "Error matching file signature", ex); //NON-NLS + return ProcessResult.ERROR; + } catch (Exception e) { + logger.log(Level.WARNING, "Error matching file signature", e); //NON-NLS + return ProcessResult.ERROR; + } + } + + /** + * @inheritDoc + */ + @Override + public void shutDown() { + /** + * If this is the instance of this module for this ingest job, post a + * summary message to the ingest messages box. + */ + if (refCounter.decrementAndGet(jobId) == 0) { + IngestJobTotals jobTotals; + synchronized (this) { + jobTotals = totalsForIngestJobs.remove(jobId); + } + if (jobTotals != null) { + StringBuilder detailsSb = new StringBuilder(); + detailsSb.append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + detailsSb.append("\n"); //NON-NLS + detailsSb.append("\n"); //NON-NLS + detailsSb.append("
    ").append(FileTypeIdModuleFactory.getModuleName()).append("
    ") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) + .append("").append(jobTotals.matchTime).append("
    ") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) + .append("").append(jobTotals.numFiles).append("
    "); //NON-NLS + IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileTypeIdIngestModule.complete.srvMsg.text"), + detailsSb.toString())); + } + } + } + + /** + * Update the match time total and increment number of files processed for + * this ingest job. + * + * @param jobId The ingest job identifier. + * @param matchTimeInc Amount of time to add. + */ + private static synchronized void addToTotals(long jobId, long matchTimeInc) { + IngestJobTotals ingestJobTotals = totalsForIngestJobs.get(jobId); + if (ingestJobTotals == null) { + ingestJobTotals = new IngestJobTotals(); + totalsForIngestJobs.put(jobId, ingestJobTotals); + } + + ingestJobTotals.matchTime += matchTimeInc; + ingestJobTotals.numFiles++; + totalsForIngestJobs.put(jobId, ingestJobTotals); + } + + private static class IngestJobTotals { + + long matchTime = 0; + long numFiles = 0; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java index 6688fd8027..38823ca03d 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java @@ -24,69 +24,121 @@ import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleFactory; import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; /** - * An factory that creates file ingest modules that determine the types of - * files. + * A factory that creates file ingest modules that determine the types of files. */ +// TODO: This class does not need to be public. @ServiceProvider(service = IngestModuleFactory.class) public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { + FileTypeIdGlobalSettingsPanel globalSettingsPanel; + + /** + * @inheritDoc + */ @Override public String getModuleDisplayName() { return getModuleName(); } + /** + * Gets the module display name. + * + * @return The name string. + */ static String getModuleName() { return NbBundle.getMessage(FileTypeIdIngestModule.class, "FileTypeIdIngestModule.moduleName.text"); } + /** + * @inheritDoc + */ @Override public String getModuleDescription() { return NbBundle.getMessage(FileTypeIdIngestModule.class, "FileTypeIdIngestModule.moduleDesc.text"); } + /** + * @inheritDoc + */ @Override public String getModuleVersionNumber() { return Version.getVersion(); } + /** + * @inheritDoc + */ + @Override + public boolean hasGlobalSettingsPanel() { + return true; + } + + /** + * @inheritDoc + */ + @Override + public IngestModuleGlobalSettingsPanel getGlobalSettingsPanel() { + if (null == globalSettingsPanel) { + globalSettingsPanel = new FileTypeIdGlobalSettingsPanel(); + } + globalSettingsPanel.load(); + return globalSettingsPanel; + } + + /** + * @inheritDoc + */ @Override public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { return new FileTypeIdModuleSettings(); } + /** + * @inheritDoc + */ @Override public boolean hasIngestJobSettingsPanel() { return true; } + /** + * @inheritDoc + */ @Override public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { assert settings instanceof FileTypeIdModuleSettings; if (!(settings instanceof FileTypeIdModuleSettings)) { throw new IllegalArgumentException(NbBundle.getMessage(this.getClass(), - "FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg")); - } - return new FileTypeIdModuleSettingsPanel((FileTypeIdModuleSettings) settings); + "FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg")); + } + return new FileTypeIdIngestJobSettingsPanel((FileTypeIdModuleSettings) settings); } + /** + * @inheritDoc + */ @Override public boolean isFileIngestModuleFactory() { return true; } + /** + * @inheritDoc + */ @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { assert settings instanceof FileTypeIdModuleSettings; if (!(settings instanceof FileTypeIdModuleSettings)) { throw new IllegalArgumentException( NbBundle.getMessage(this.getClass(), "FileTypeIdModuleFactory.createFileIngestModule.exception.msg")); - } + } return new FileTypeIdIngestModule((FileTypeIdModuleSettings) settings); } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java index c472596be4..270c604d25 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java @@ -23,23 +23,30 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; /** * Ingest job options for the file type identifier ingest module instances. */ +// TODO: This class does not need to be public. public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { private static final long serialVersionUID = 1L; - private volatile boolean skipKnownFiles = true; + private static final long MIN_FILE_SIZE_IN_BYTES = 512; + private boolean skipKnownFiles = true; + private boolean skipSmallFiles = true; FileTypeIdModuleSettings() { } - FileTypeIdModuleSettings(boolean skipKnownFiles) { + FileTypeIdModuleSettings(boolean skipKnownFiles, boolean skipSmallFiles) { this.skipKnownFiles = skipKnownFiles; + this.skipSmallFiles = skipSmallFiles; } - + + /** + * @inheritDoc + */ @Override public long getVersionNumber() { return serialVersionUID; - } - + } + void setSkipKnownFiles(boolean enabled) { skipKnownFiles = enabled; } @@ -47,4 +54,17 @@ public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { boolean skipKnownFiles() { return skipKnownFiles; } -} \ No newline at end of file + + void setSkipSmallFiles(boolean enabled) { + this.skipSmallFiles = enabled; + } + + boolean skipSmallFiles() { + return skipSmallFiles; + } + + long minFileSizeInBytes() { + return MIN_FILE_SIZE_IN_BYTES; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java new file mode 100644 index 0000000000..a78abb0c5c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java @@ -0,0 +1,90 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.beans.PropertyChangeListener; +import java.beans.PropertyChangeSupport; +import javax.swing.JComponent; +import org.netbeans.spi.options.OptionsPanelController; +import org.openide.util.HelpCtx; +import org.openide.util.Lookup; + +@OptionsPanelController.TopLevelRegistration( + categoryName = "#OptionsCategory_Name_FileTypeId", + iconBase = "org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png", + keywords = "#OptionsCategory_Keywords_FileTypeId", + keywordsCategory = "FileTypeId", + position = 5 +) +@org.openide.util.NbBundle.Messages({"OptionsCategory_Name_FileTypeId=FileTypeId", "OptionsCategory_Keywords_FileTypeId=FileTypeId"}) +public final class FileTypeIdOptionsPanelController extends OptionsPanelController { + + private FileTypeIdGlobalSettingsPanel panel; + private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); + private boolean changed; + + @Override + public void update() { + getPanel().load(); + changed = false; + } + + @Override + public void applyChanges() { + getPanel().store(); + changed = false; + } + + @Override + public void cancel() { + } + + @Override + public boolean isValid() { + return true; + } + + @Override + public boolean isChanged() { + return changed; + } + + @Override + public HelpCtx getHelpCtx() { + return null; + } + + @Override + public JComponent getComponent(Lookup masterLookup) { + return getPanel(); + } + + @Override + public void addPropertyChangeListener(PropertyChangeListener l) { + pcs.addPropertyChangeListener(l); + } + + @Override + public void removePropertyChangeListener(PropertyChangeListener l) { + pcs.removePropertyChangeListener(l); + } + + private FileTypeIdGlobalSettingsPanel getPanel() { + if (panel == null) { + panel = new FileTypeIdGlobalSettingsPanel(); + } + return panel; + } + + void changed() { + if (!changed) { + changed = true; + pcs.firePropertyChange(OptionsPanelController.PROP_CHANGED, false, true); + } + pcs.firePropertyChange(OptionsPanelController.PROP_VALID, null, null); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypes.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypes.xsd new file mode 100644 index 0000000000..26aa720ff1 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypes.xsd @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java index a6ba0e5483..84d1434644 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java @@ -69,18 +69,6 @@ public class TikaFileTypeDetector { buf = buffer; } - // the xml detection in Tika tries to parse the entire file and throws exceptions - // for files that are not valid XML - try { - String tagHeader = new String(buf, 0, 5); - if (tagHeader.equals(" sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.util.HashMap; +import java.util.Map; +import org.sleuthkit.datamodel.AbstractFile; + +/** + * Does file type detection for user-defined file types. + */ +final class UserDefinedFileTypeDetector { + + private Map fileTypes; + + /** + * Creates an object that can do file type detection for user-defined + * file types. + */ + UserDefinedFileTypeDetector() { + fileTypes = new HashMap<>(); + } + + /** + * Gets the user-defined file types from the user-defined file types + * manager. + * + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + void loadFileTypes() throws UserDefinedFileTypesManager.UserDefinedFileTypesException { + fileTypes = UserDefinedFileTypesManager.getInstance().getFileTypes(); + } + + /** + * Attempts to identify a file using the set of user-defined file type file + * types. + * + * @param file The file to type. + * @return A FileType object or null if identification fails. + */ + FileType identify(final AbstractFile file) { + FileType type = null; + for (FileType fileType : this.fileTypes.values()) { + if (fileType.matches(file)) { + type = fileType; + break; + } + } + return type; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java new file mode 100644 index 0000000000..1884330707 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -0,0 +1,518 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import javax.xml.bind.DatatypeConverter; +import javax.xml.transform.TransformerException; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; +import org.xml.sax.SAXException; + +/** + * Manages user-defined file types characterized by MIME type, signature, and + * optional membership in an interesting files set. + *

    + * Note that this class exposes a very simple get/set API that operates on the + * user-defined file types as a complete set - there is no concept of adding, + * editing or deleting file types singly. This works because this class is not + * exposed outside of this ingest module package and is ONLY used in a very + * specific paradigm. In this paradigm, there is a single modal writer of file + * types in the form of a global settings panel that disables itself when ingest + * is running so that multiple readers in the form of file ingest modules get a + * consistent set of file type definitions. + *

    + * Thread-safe. + */ +final class UserDefinedFileTypesManager { + + private static final Logger logger = Logger.getLogger(UserDefinedFileTypesManager.class.getName()); + private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypes.xsd"; //NON-NLS + private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; //NON-NLS + private static final String FILE_TYPES_TAG_NAME = "FileTypes"; //NON-NLS + private static final String FILE_TYPE_TAG_NAME = "FileType"; //NON-NLS + private static final String MIME_TYPE_TAG_NAME = "MimeType"; //NON-NLS + private static final String SIGNATURE_TAG_NAME = "Signature"; //NON-NLS + private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; //NON-NLS + private static final String BYTES_TAG_NAME = "Bytes"; //NON-NLS + private static final String OFFSET_TAG_NAME = "Offset"; //NON-NLS + private static final String INTERESTING_FILES_SET_TAG_NAME = "InterestingFileSset"; //NON-NLS + private static final String ALERT_ATTRIBUTE = "alert"; //NON-NLS + private static final String ENCODING_FOR_XML_FILE = "UTF-8"; //NON-NLS + private static final String ASCII_ENCODING = "US-ASCII"; //NON-NLS + private static UserDefinedFileTypesManager instance; + + /** + * File types to be persisted to the user-defined file type definitions file + * are stored in this mapping of MIME types to file types. Access to this + * map is guarded by the intrinsic lock of the user-defined file types + * manager for thread-safety. + */ + private final Map userDefinedFileTypes = new HashMap<>(); + + /** + * The combined set of user-defined file types and file types predefined by + * Autopsy are stored in this mapping of MIME types to file types. This is + * the current working set of file types. Access to this map is guarded by + * the intrinsic lock of the user-defined file types manager for + * thread-safety. + */ + private final Map fileTypes = new HashMap<>(); + + /** + * Gets the singleton manager of user-defined file types characterized by + * MIME type, signature, and optional membership in an interesting files + * set. + * + * @return The user-defined file types manager singleton. + */ + synchronized static UserDefinedFileTypesManager getInstance() { + if (instance == null) { + instance = new UserDefinedFileTypesManager(); + } + return instance; + } + + /** + * Creates a manager of user-defined file types characterized by MIME type, + * signature, and optional membership in an interesting files set. + */ + private UserDefinedFileTypesManager() { + } + + /** + * Gets both the predefined and the user-defined file types. + * + * @return A mapping of file type names to file types, possibly empty. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized Map getFileTypes() throws UserDefinedFileTypesException { + loadFileTypes(); + + /** + * It is safe to return references to the internal file type objects + * because they are immutable. Note that + * Collections.unmodifiableCollection() is not used here because this + * view of the file types is a snapshot. + */ + return new HashMap<>(fileTypes); + } + + /** + * Gets the user-defined file types. + * + * @return A mapping of file type names to file types, possibly empty. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized Map getUserDefinedFileTypes() throws UserDefinedFileTypesException { + loadFileTypes(); + + /** + * It is safe to return references to the internal file type objects + * because they are immutable. Note that + * Collections.unmodifiableCollection() is not used here because this + * view of the file types is a snapshot. + */ + return new HashMap<>(userDefinedFileTypes); + } + + /** + * Loads the MIME type to file type mappings with predefined and + * user-defined types. + * + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + private void loadFileTypes() throws UserDefinedFileTypesException { + fileTypes.clear(); + userDefinedFileTypes.clear(); + /** + * Load the predefined types first so that they can be overwritten by + * any user-defined types with the same names. + */ + loadPredefinedFileTypes(); + loadUserDefinedFileTypes(); + } + + /** + * Adds the predefined file types to the in-memory mappings of MIME types to + * file types. + * + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + private void loadPredefinedFileTypes() throws UserDefinedFileTypesException { + try { + FileType fileType = new FileType("text/xml", new Signature(" newFileTypes) throws UserDefinedFileTypesException { + try { + String filePath = getFileTypeDefinitionsFilePath(USER_DEFINED_TYPE_DEFINITIONS_FILE); + XmlWriter.writeFileTypes(newFileTypes.values(), filePath); + } catch (ParserConfigurationException | FileNotFoundException | UnsupportedEncodingException | TransformerException ex) { + throwUserDefinedFileTypesException(ex, "UserDefinedFileTypesManager.saveFileTypes.errorMessage"); + } catch (IOException ex) { + throwUserDefinedFileTypesException(ex, "UserDefinedFileTypesManager.saveFileTypes.errorMessage"); + } + } + + /** + * Gets the absolute path of a file type definitions file. + * + * @param fileName The name of the file. + * @return The absolute path to the file. + */ + private static String getFileTypeDefinitionsFilePath(String fileName) { + Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), fileName); + return filePath.toAbsolutePath().toString(); + } + + /** + * Provides a mechanism for writing a set of file type definitions to an XML + * file. + */ + private static class XmlWriter { + + /** + * Writes a set of file type definitions to an XML file. + * + * @param fileTypes A collection of file types. + * @param filePath The path to the destination file. + * @throws ParserConfigurationException + * @throws IOException + * @throws FileNotFoundException + * @throws UnsupportedEncodingException + * @throws TransformerException + */ + private static void writeFileTypes(Collection fileTypes, String filePath) throws ParserConfigurationException, IOException, FileNotFoundException, UnsupportedEncodingException, TransformerException { + Document doc = XMLUtil.createDocument(); + Element fileTypesElem = doc.createElement(FILE_TYPES_TAG_NAME); + doc.appendChild(fileTypesElem); + for (FileType fileType : fileTypes) { + Element fileTypeElem = XmlWriter.createFileTypeElement(fileType, doc); + fileTypesElem.appendChild(fileTypeElem); + } + XMLUtil.saveDocument(doc, ENCODING_FOR_XML_FILE, filePath); + } + + /** + * Creates an XML representation of a file type. + * + * @param fileType The file type object. + * @param doc The WC3 DOM object to use to create the XML. + * @return An XML element. + */ + private static Element createFileTypeElement(FileType fileType, Document doc) { + Element fileTypeElem = doc.createElement(FILE_TYPE_TAG_NAME); + XmlWriter.addMimeTypeElement(fileType, fileTypeElem, doc); + XmlWriter.addSignatureElement(fileType, fileTypeElem, doc); + XmlWriter.addInterestingFilesSetElement(fileType, fileTypeElem, doc); + XmlWriter.addAlertAttribute(fileType, fileTypeElem); + return fileTypeElem; + } + + /** + * Add a MIME type child element to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + * @param doc The WC3 DOM object to use to create the XML. + */ + private static void addMimeTypeElement(FileType fileType, Element fileTypeElem, Document doc) { + Element typeNameElem = doc.createElement(MIME_TYPE_TAG_NAME); + typeNameElem.setTextContent(fileType.getMimeType()); + fileTypeElem.appendChild(typeNameElem); + } + + /** + * Add a signature child element to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + * @param doc The WC3 DOM object to use to create the XML. + */ + private static void addSignatureElement(FileType fileType, Element fileTypeElem, Document doc) { + Signature signature = fileType.getSignature(); + Element signatureElem = doc.createElement(SIGNATURE_TAG_NAME); + + Element bytesElem = doc.createElement(BYTES_TAG_NAME); + bytesElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); + signatureElem.appendChild(bytesElem); + + Element offsetElem = doc.createElement(OFFSET_TAG_NAME); + offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); + signatureElem.appendChild(offsetElem); + + signatureElem.setAttribute(SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); + fileTypeElem.appendChild(signatureElem); + } + + /** + * Add an interesting files set element to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + * @param doc The WC3 DOM object to use to create the XML. + */ + private static void addInterestingFilesSetElement(FileType fileType, Element fileTypeElem, Document doc) { + if (!fileType.getFilesSetName().isEmpty()) { + Element filesSetElem = doc.createElement(INTERESTING_FILES_SET_TAG_NAME); + filesSetElem.setTextContent(fileType.getFilesSetName()); + fileTypeElem.appendChild(filesSetElem); + } + } + + /** + * Add an alert attribute to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + */ + private static void addAlertAttribute(FileType fileType, Element fileTypeElem) { + fileTypeElem.setAttribute(ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); + } + + } + + /** + * Provides a mechanism for reading a set of file type definitions from an + * XML file. + */ + private static class XmlReader { + + /** + * Reads a set of file type definitions from an XML file. + * + * @param filePath The path to the XML file. + * @return A collection of file types read from the XML file. + */ + private static List readFileTypes(String filePath) throws IOException, ParserConfigurationException, SAXException { + List fileTypes = new ArrayList<>(); + Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.class, FILE_TYPE_DEFINITIONS_SCHEMA_FILE); + if (doc != null) { + Element fileTypesElem = doc.getDocumentElement(); + if (fileTypesElem != null && fileTypesElem.getNodeName().equals(FILE_TYPES_TAG_NAME)) { + NodeList fileTypeElems = fileTypesElem.getElementsByTagName(FILE_TYPE_TAG_NAME); + for (int i = 0; i < fileTypeElems.getLength(); ++i) { + Element fileTypeElem = (Element) fileTypeElems.item(i); + FileType fileType = XmlReader.parseFileType(fileTypeElem); + fileTypes.add(fileType); + } + } + } + return fileTypes; + } + + /** + * Gets a file type definition from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return A file type object. + * @throws IllegalArgumentException + * @throws NumberFormatException + */ + private static FileType parseFileType(Element fileTypeElem) throws IllegalArgumentException, NumberFormatException { + String mimeType = XmlReader.parseMimeType(fileTypeElem); + Signature signature = XmlReader.parseSignature(fileTypeElem); + String filesSetName = XmlReader.parseInterestingFilesSet(fileTypeElem); + boolean alert = XmlReader.parseAlert(fileTypeElem); + return new FileType(mimeType, signature, filesSetName, alert); + } + + /** + * Gets the MIME type from a file type XML element. + * + * @param fileTypeElem The element + * @return A MIME type string. + */ + private static String parseMimeType(Element fileTypeElem) { + return getChildElementTextContent(fileTypeElem, MIME_TYPE_TAG_NAME); + } + + /** + * Gets the signature from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return The signature. + */ + private static Signature parseSignature(Element fileTypeElem) throws IllegalArgumentException, NumberFormatException { + NodeList signatureElems = fileTypeElem.getElementsByTagName(SIGNATURE_TAG_NAME); + Element signatureElem = (Element) signatureElems.item(0); + + String sigTypeAttribute = signatureElem.getAttribute(SIGNATURE_TYPE_ATTRIBUTE); + Signature.Type signatureType = Signature.Type.valueOf(sigTypeAttribute); + + String sigBytesString = getChildElementTextContent(signatureElem, BYTES_TAG_NAME); + byte[] signatureBytes = DatatypeConverter.parseHexBinary(sigBytesString); + + String offsetString = getChildElementTextContent(signatureElem, OFFSET_TAG_NAME); + long offset = DatatypeConverter.parseLong(offsetString); + + return new Signature(signatureBytes, offset, signatureType); + } + + /** + * Gets the interesting files set name from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return The files set name, possibly empty. + */ + private static String parseInterestingFilesSet(Element fileTypeElem) { + String filesSetName = ""; + NodeList filesSetElems = fileTypeElem.getElementsByTagName(INTERESTING_FILES_SET_TAG_NAME); + if (filesSetElems.getLength() > 0) { + Element filesSetElem = (Element) filesSetElems.item(0); + filesSetName = filesSetElem.getTextContent(); + } + return filesSetName; + } + + /** + * Gets the alert attribute from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return True or false; + */ + private static boolean parseAlert(Element fileTypeElem) { + String alertAttribute = fileTypeElem.getAttribute(ALERT_ATTRIBUTE); + return Boolean.parseBoolean(alertAttribute); + } + + /** + * Gets the text content of a single child element. + * + * @param elem The parent element. + * @param tagName The tag name of the child element. + * @return The text content. + */ + private static String getChildElementTextContent(Element elem, String tagName) { + NodeList childElems = elem.getElementsByTagName(tagName); + Element childElem = (Element) childElems.item(0); + return childElem.getTextContent(); + } + + } + + /** + * Logs an exception, bundles the exception with a simple message in a + * uniform exception type, and throws the wrapper exception. + * + * @param ex The exception to wrap. + * @param messageKey A key into the bundle file that maps to the desired + * message. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + private void throwUserDefinedFileTypesException(Exception ex, String messageKey) throws UserDefinedFileTypesException { + String message = NbBundle.getMessage(UserDefinedFileTypesManager.class, messageKey); + logger.log(Level.SEVERE, message, ex); + throw new UserDefinedFileTypesException(message, ex); + } + + /** + * Used to translate more implementation-details-specific exceptions (which + * are logged by this class) into more generic exceptions for propagation to + * clients of the user-defined file types manager. + */ + static class UserDefinedFileTypesException extends Exception { + + UserDefinedFileTypesException(String message) { + super(message); + } + + UserDefinedFileTypesException(String message, Throwable throwable) { + super(message, throwable); + } + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png new file mode 100644 index 0000000000..4dc0c47b17 Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png differ diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/warning16.png b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/warning16.png new file mode 100644 index 0000000000..f5ba881738 Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/warning16.png differ diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties new file mode 100644 index 0000000000..38b86e4f10 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties @@ -0,0 +1,2 @@ +iOSModuleFactory.moduleDescription=\u30B7\u30B9\u30C6\u30E0\u304A\u3088\u3073\u7B2C\u4E09\u8005\u30A2\u30D7\u30EA\u30C7\u30FC\u30BF\u3092\u62BD\u51FA +iOSModuleFactory.moduleName=iOS\u30A2\u30CA\u30E9\u30A4\u30B6 \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java index 38831120e0..a3d8cc44ba 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java @@ -49,7 +49,7 @@ class CallLogAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NON-NLS //get exact file names if (absFiles.isEmpty()) { return; } @@ -61,11 +61,11 @@ class CallLogAnalyzer { fileId = AF.getId(); findCallLogsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs", e); + logger.log(Level.SEVERE, "Error parsing Call logs", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Call logs", e); + logger.log(Level.SEVERE, "Error finding Call logs", e); //NON-NLS } } @@ -74,11 +74,11 @@ class CallLogAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS } Case currentCase = Case.getCurrentCase(); @@ -87,7 +87,7 @@ class CallLogAnalyzer { AbstractFile f = skCase.getAbstractFileById(fId); try { resultSet = statement.executeQuery( - "SELECT number,date,duration,type, name FROM calls ORDER BY date DESC;"); + "SELECT number,date,duration,type, name FROM calls ORDER BY date DESC;"); //NON-NLS BlackboardArtifact bba; String name; // name of person dialed or called. null if unregistered @@ -97,14 +97,14 @@ class CallLogAnalyzer { String type; // 1 incoming, 2 outgoing, 3 missed while (resultSet.next()) { - name = resultSet.getString("name"); - number = resultSet.getString("number"); - duration = resultSet.getString("duration"); - date = resultSet.getString("date"); - type = resultSet.getString("type"); + name = resultSet.getString("name"); //NON-NLS + number = resultSet.getString("number"); //NON-NLS + duration = resultSet.getString("duration"); //NON-NLS + date = resultSet.getString("date"); //NON-NLS + type = resultSet.getString("type"); //NON-NLS bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); //create a call log and then add attributes from result set. - if(type.equalsIgnoreCase("outgoing")) { + if(type.equalsIgnoreCase("outgoing")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO.getTypeID(), moduleName, number)); } else { /// Covers INCOMING and MISSED @@ -117,18 +117,18 @@ class CallLogAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NON-NLS } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing the database", e); + logger.log(Level.SEVERE, "Error closing the database", e); //NON-NLS } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java index 0e7ee51e76..becafbafc0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java @@ -56,7 +56,7 @@ class ContactAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name LIKE '%call_history%' "); //get exact file names + absFiles = skCase.findAllFilesWhere("name LIKE '%call_history%' "); //NON-NLS //get exact file names if (absFiles.isEmpty()) { //asdfkjasfakljsdfhlaksdjfhasdlkjf return; } @@ -71,11 +71,11 @@ class ContactAnalyzer { fileId = AF.getId(); //findContactsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts", e); + logger.log(Level.SEVERE, "Error parsing Contacts", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Contacts", e); + logger.log(Level.SEVERE, "Error finding Contacts", e); //NON-NLS } } @@ -90,11 +90,11 @@ class ContactAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS } Case currentCase = Case.getCurrentCase(); @@ -105,13 +105,13 @@ class ContactAnalyzer { // get display_name, mimetype(email or phone number) and data1 (phonenumber or email address depending on mimetype) //sorted by name, so phonenumber/email would be consecutive for a person if they exist. resultSet = statement.executeQuery( - "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" - + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" - + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " - + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" - + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" - + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" - + "ORDER BY name_raw_contact.display_name ASC;"); + "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NON-NLS + + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NON-NLS + + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NON-NLS + + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NON-NLS + + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NON-NLS + + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NON-NLS + + "ORDER BY name_raw_contact.display_name ASC;"); //NON-NLS BlackboardArtifact bba; bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); @@ -120,15 +120,15 @@ class ContactAnalyzer { String mimetype; // either phone or email String data1; // the phone number or email while (resultSet.next()) { - name = resultSet.getString("display_name"); - data1 = resultSet.getString("data1"); - mimetype = resultSet.getString("mimetype"); + name = resultSet.getString("display_name"); //NON-NLS + data1 = resultSet.getString("data1"); //NON-NLS + mimetype = resultSet.getString("mimetype"); //NON-NLS // System.out.println(resultSet.getString("data1") + resultSet.getString("mimetype") + resultSet.getString("display_name")); //Test code if (name.equals(oldName) == false) { bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } - if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { + if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getTypeID(), moduleName, data1)); } else { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getTypeID(), moduleName, data1)); @@ -137,18 +137,18 @@ class ContactAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NON-NLS } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NON-NLS } } @@ -186,13 +186,13 @@ class ContactAnalyzer { ostream.write(c); } } catch (IOException e) { - System.out.println("Error: " + e.getMessage()); + System.out.println("Error: " + e.getMessage()); //NON-NLS } finally { try { istream.close(); ostream.close(); } catch (IOException e) { - System.out.println("File did not close"); + System.out.println("File did not close"); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java index 434bcd0ea0..04d67b78f7 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java @@ -49,7 +49,7 @@ class TextMessageAnalyzer { void findTexts() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //get exact file name + absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NON-NLS //get exact file name if (absFiles.isEmpty()) { return; } @@ -61,11 +61,11 @@ class TextMessageAnalyzer { fileId = AF.getId(); findTextsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages", e); + logger.log(Level.SEVERE, "Error parsing text messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding text messages", e); + logger.log(Level.SEVERE, "Error finding text messages", e); //NON-NLS } } @@ -74,11 +74,11 @@ class TextMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS } Case currentCase = Case.getCurrentCase(); @@ -87,7 +87,7 @@ class TextMessageAnalyzer { AbstractFile f = skCase.getAbstractFileById(fId); try { resultSet = statement.executeQuery( - "Select address,date,type,subject,body FROM sms;"); + "Select address,date,type,subject,body FROM sms;"); //NON-NLS BlackboardArtifact bba; String address; // may be phone number, or other addresses @@ -96,11 +96,11 @@ class TextMessageAnalyzer { String subject;//message subject String body; //message body while (resultSet.next()) { - address = resultSet.getString("address"); - date = resultSet.getString("date"); - type = resultSet.getString("type"); - subject = resultSet.getString("subject"); - body = resultSet.getString("body"); + address = resultSet.getString("address"); //NON-NLS + date = resultSet.getString("date"); //NON-NLS + type = resultSet.getString("type"); //NON-NLS + subject = resultSet.getString("subject"); //NON-NLS + body = resultSet.getString("body"); //NON-NLS bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create Message artifact and then add attributes from result set. @@ -122,18 +122,18 @@ class TextMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NON-NLS } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties index 68f5433dc1..7a25a4305c 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties @@ -2,12 +2,10 @@ OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=Interesting Files Identifier ingest module. \n\n Identifies interesting files as defined by interesting files rule sets. OpenIDE-Module-Short-Description=Interesting Files Identifier ingest module. OpenIDE-Module-Name=Interesting Files Identifier +OptionsCategory_Name_InterestingItemDefinitions=Interesting Item Definitions +OptionsCategory_Keywords_InterestingItemDefinitions=InterestingItemDefinitions InterestingItemsIdentifierIngestModule.moduleName=Interesting Files Identifier InterestingItemsIdentifierIngestModule.moduleDescription=Identifies interesting items as defined by interesting item rule sets. -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title0=Title 1 -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title3=Title 4 -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title2=Title 3 -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title1=Title 2 InterestingItemDefsPanel.newSetButton.text=New Set InterestingItemDefsPanel.editSetButton.text=Edit Set InterestingItemDefsPanel.deleteSetButton.text=Delete Set diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index e69de29bb2..ae17fa6dc4 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -0,0 +1,56 @@ +FilesIdentifierIngestJobSettingsPanel.border.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u4E2D\u6709\u52B9\u306B\u3059\u308B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u3092\u9078\u629E\uFF1A +FilesSetPanel.descPanel.border.title=\u6982\u8981 +FilesSetPanel.descriptionPanel.border.title=\u6982\u8981 +FilesSetPanel.ignoreKnownFilesCheckbox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\u3092\u7121\u8996 +FilesSetPanel.messages.filesSetsMustBeNamed=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u306F\u540D\u524D\u304C\u5FC5\u8981\u3067\u3059\u3002 +FilesSetPanel.nameLabel.text=\u30BB\u30C3\u30C8\u540D\uFF1A +FilesSetPanel.title=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8 +FilesSetRulePanel.dirsRadioButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +FilesSetRulePanel.extensionRadioButton.text=\u62E1\u5F35\u5B50\u306E\u307F +FilesSetRulePanel.filesAndDirsRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u304A\u3088\u3073\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +FilesSetRulePanel.filesRadioButton.text=\u30D5\u30A1\u30A4\u30EB +FilesSetRulePanel.fullNameRadioButton.text=\u30D5\u30EB\u30CD\u30FC\u30E0 +FilesSetRulePanel.jLabel1.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\uFF1A +FilesSetRulePanel.jLabel2.text=\u30CD\u30FC\u30E0\u30D1\u30BF\u30FC\u30F3\uFF1A +FilesSetRulePanel.jLabel3.text=\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\uFF1A +FilesSetRulePanel.messages.emptyNameFilter=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u30EB\u30FC\u30EB\u306F\u691C\u7D22\u3059\u308B\u540D\u524D\u304C\u5FC5\u8981\u3067\u3059\u3002 +FilesSetRulePanel.messages.filesSetRulesMustBeNamed=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u30EB\u30FC\u30EB\u306F\u540D\u524D\u304C\u5FC5\u8981\u3067\u3059\u3002 +FilesSetRulePanel.messages.invalidCharInName=\u6B63\u898F\u8868\u73FE\u4EE5\u5916\u306F\\\u3001/\u3001\:\u3001*\u3001?\u3001"\u3001<\u3001>\u3092\u540D\u524D\u306B\u542B\u3081\u307E\u305B\u3093\u3002 +FilesSetRulePanel.messages.invalidCharInPath=\u6B63\u898F\u8868\u73FE\u4EE5\u5916\u306F\\\u3001\:\u3001*\u3001?\u3001"\u3001<\u3001>\u3092\u30D1\u30B9\u306B\u542B\u3081\u307E\u305B\u3093\u3002 +FilesSetRulePanel.messages.invalidNameRegex=\u6B63\u898F\u8868\u73FE\u306F\u6709\u52B9\u306A\u540D\u524D\u3067\u306F\u3042\u308A\u307E\u305B\u3093\uFF1A\n\n{0} +FilesSetRulePanel.messages.invalidPathRegex=\u6B63\u898F\u8868\u73FE\u306F\u6709\u52B9\u306A\u30D1\u30B9\u3067\u306F\u3042\u308A\u307E\u305B\u3093\uFF1A\n\n{0} +FilesSetRulePanel.nameRegexCheckbox.text=\u6B63\u898F\u8868\u73FE +FilesSetRulePanel.pathRegexCheckBox.text=\u6B63\u898F\u8868\u73FE +FilesSetRulePanel.pathSeparatorInfoLabel.text=/\u3092\u30D1\u30B9\u533A\u5207\u308A\u6587\u5B57\u3068\u3057\u3066\u5229\u7528 +FilesSetRulePanel.ruleNameLabel.text=\u30EB\u30FC\u30EB\u540D\uFF1A +FilesSetRulePanel.title=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u30EB\u30FC\u30EB +InterestingItemDefsPanel.bothRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u304A\u3088\u3073\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +InterestingItemDefsPanel.deleteRuleButton.text=\u30EB\u30FC\u30EB\u3092\u524A\u9664 +InterestingItemDefsPanel.deleteSetButton.text=\u30BB\u30C3\u30C8\u3092\u524A\u9664 +InterestingItemDefsPanel.dirsRadioButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +InterestingItemDefsPanel.editRuleButton.text=\u30EB\u30FC\u30EB\u3092\u7DE8\u96C6 +InterestingItemDefsPanel.editSetButton.text=\u30BB\u30C3\u30C8\u3092\u7DE8\u96C6 +InterestingItemDefsPanel.fileNameExtensionRadioButton.text=\u62E1\u5F35\u5B50\u306E\u307F +InterestingItemDefsPanel.fileNameRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u540D +InterestingItemDefsPanel.fileNameRegexCheckbox.text=\u6B63\u898F\u8868\u73FE +InterestingItemDefsPanel.filesRadioButton.text=\u30D5\u30A1\u30A4\u30EB +InterestingItemDefsPanel.ignoreKnownFilesCheckbox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\u3092\u7121\u8996 +InterestingItemDefsPanel.jLabel1.text=\u30EB\u30FC\u30EB\u8A73\u7D30 +InterestingItemDefsPanel.jLabel2.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\uFF1A +InterestingItemDefsPanel.jLabel3.text=\u30CD\u30FC\u30E0\u30D1\u30BF\u30FC\u30F3 +InterestingItemDefsPanel.jLabel4.text=\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\uFF1A +InterestingItemDefsPanel.jLabel5.text=\u6982\u8981\uFF1A +InterestingItemDefsPanel.jLabel6.text=\u30BB\u30C3\u30C8\u8A73\u7D30 +InterestingItemDefsPanel.jTextArea1.text=\u6307\u5B9A\u3055\u308C\u305F\u6761\u4EF6\u3068\u4E00\u81F4\u3059\u308B\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u7D22\u3059\u308B\u306E\u304C\u53EF\u80FD\u306A\u30E2\u30B8\u30E5\u30FC\u30EB\u3067\u3059\u3002\u5404\u30BB\u30C3\u30C8\u306B\u306F\u30D5\u30A1\u30A4\u30EB\u540D\u304A\u3088\u3073\u30DA\u30A2\u30EC\u30F3\u30C8\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\u3092\u3082\u3068\u306B\u4E00\u81F4\u3059\u308B\u3001\u30EB\u30FC\u30EB\u30EA\u30B9\u30C8\u304C\u3042\u308A\u307E\u3059\u3002 +InterestingItemDefsPanel.newRuleButton.text=\u65B0\u898F\u30EB\u30FC\u30EB +InterestingItemDefsPanel.newSetButton.text=\u65B0\u898F\u30BB\u30C3\u30C8 +InterestingItemDefsPanel.rulePathFilterRegexCheckBox.text=\u6B63\u898F\u8868\u73FE +InterestingItemDefsPanel.rulesListLabel.text=\u30EB\u30FC\u30EB\uFF1A +InterestingItemDefsPanel.setsListLabel.text=\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8 +InterestingItemsIdentifierIngestModule.moduleDescription=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u3092\u898B\u3064\u3051\u307E\u3059\u3002 +InterestingItemsIdentifierIngestModule.moduleName=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 +OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OpenIDE-Module-Long-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u77E5\u3057\u307E\u3059\u3002 +OpenIDE-Module-Name=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 +OpenIDE-Module-Short-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OptionsCategory_Name_InterestingItemDefinitions=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u5B9A\u7FA9 \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java index 0643a5f722..3b858680f5 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java @@ -109,7 +109,7 @@ final class FilesIdentifierIngestModule implements FileIngestModule { artifact.addAttribute(ruleNameAttribute); } catch (TskCoreException ex) { - FilesIdentifierIngestModule.logger.log(Level.SEVERE, "Error posting to the blackboard", ex); + FilesIdentifierIngestModule.logger.log(Level.SEVERE, "Error posting to the blackboard", ex); //NOI18N } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java index 2f1c31caba..735683cc58 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java @@ -255,11 +255,11 @@ final class InterestingItemDefsManager extends Observable { if (!rules.containsKey(rule.getName())) { rules.put(rule.getName(), rule); } else { - logger.log(Level.SEVERE, "Found duplicate rule {0} for set named {1} in interesting file sets definition file at {2}, discarding malformed set", new Object[]{rule.getName(), setName, filePath}); + logger.log(Level.SEVERE, "Found duplicate rule {0} for set named {1} in interesting file sets definition file at {2}, discarding malformed set", new Object[]{rule.getName(), setName, filePath}); //NOI18N return; } } else { - logger.log(Level.SEVERE, "Found malformed rule for set named {0} in interesting file sets definition file at {1}, discarding malformed set", new Object[]{setName, filePath}); + logger.log(Level.SEVERE, "Found malformed rule for set named {0} in interesting file sets definition file at {1}, discarding malformed set", new Object[]{setName, filePath}); //NOI18N return; } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java index c95bffb053..b23ecb8dc0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java @@ -31,9 +31,8 @@ import org.openide.util.Lookup; iconBase = "org/sleuthkit/autopsy/images/interesting_item_32x32.png", keywords = "#OptionsCategory_Keywords_InterestingItemDefinitions", keywordsCategory = "InterestingItemDefinitions", - position = 5 + position = 6 ) -@org.openide.util.NbBundle.Messages({"OptionsCategory_Name_InterestingItemDefinitions=Interesting Item Definitions", "OptionsCategory_Keywords_InterestingItemDefinitions=InterestingItemDefinitions"}) public final class InterestingItemDefsOptionsPanelController extends OptionsPanelController { private InterestingItemDefsPanel panel; diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java index 3eddc9236d..a52f87ffd0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.modules.interestingitems; import java.awt.EventQueue; +import java.awt.Font; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -532,7 +533,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp jTextArea1.setBackground(new java.awt.Color(240, 240, 240)); jTextArea1.setColumns(20); - jTextArea1.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N + jTextArea1.setFont(jTextArea1.getFont().deriveFont(Font.PLAIN, 11)); // NON-NLS jTextArea1.setLineWrap(true); jTextArea1.setRows(3); jTextArea1.setText(org.openide.util.NbBundle.getMessage(InterestingItemDefsPanel.class, "InterestingItemDefsPanel.jTextArea1.text")); // NOI18N diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties index 81d6260109..b9a9d6c609 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties @@ -2,9 +2,10 @@ OpenIDE-Module-Name=PhotoRec Carver Ingest Module OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=PhotoRec Carver ingest module. \n\n Carves unallocated space and feeds the resulting carved files back into the system for processing. OpenIDE-Module-Short-Description=Carves unallocated space and feeds carved files back into the system for processing. -unallocatedSpaceProcessingSettingsError.message="Process Unallocated Space" is not checked. This module is designed to carve unallocated space. Either allow processing of unallocated space, or do not use this module. moduleDisplayName.text=PhotoRec Carver moduleDescription.text=Runs PhotoRec carver against unallocated space on the system. + +unallocatedSpaceProcessingSettingsError.message="Process Unallocated Space" is not checked. This module is designed to carve unallocated space. Either allow processing of unallocated space, or do not use this module. unsupportedOS.message=Module is not supported for other than Windows platforms missingExecutable.message=Unable to locate unallocated carver executable. cannotRunExecutable.message=Unable to execute unallocated carver diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties new file mode 100644 index 0000000000..45cdd48789 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties @@ -0,0 +1,6 @@ +moduleDescription.text=\u30B7\u30B9\u30C6\u30E0\u306E\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u306B\u5BFE\u3057\u3066PhotoRec\u30AB\u30FC\u30D0\u3092\u5B9F\u884C\u3057\u307E\u3059\u3002 +moduleDisplayName.text=PhotoRec\u30AB\u30FC\u30D0 +OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OpenIDE-Module-Long-Description=PhotoRec\u30AB\u30FC\u30D0\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u5207\u308A\u51FA\u3057\u3001\u51E6\u7406\u3059\u308B\u3081\u306B\u30B7\u30B9\u30C6\u30E0\u3078\u30D5\u30A3\u30FC\u30C9\u3057\u307E\u3059\u3002 +OpenIDE-Module-Name=PhotoRec\u30AB\u30FC\u30D0\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OpenIDE-Module-Short-Description=\u51E6\u7406\u3059\u308B\u3081\u306B\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u5207\u308A\u51FA\u3057\u3001\u30B7\u30B9\u30C6\u30E0\u3078\u30D5\u30A3\u30FC\u30C9\u3057\u307E\u3059\u3002 \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index ffb5447804..cd5723e377 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -113,7 +113,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { PhotoRecCarverFileIngestModule.pathsByJob.put(this.context.getJobId(), new WorkingPaths(outputDirPath, tempDirPath)); } catch (SecurityException | IOException | UnsupportedOperationException ex) { - throw new IngestModule.IngestModuleException(NbBundle.getMessage(this.getClass(), "Utilities.cannotCreateOutputDir.message", ex.getLocalizedMessage())); + throw new IngestModule.IngestModuleException(NbBundle.getMessage(this.getClass(), "cannotCreateOutputDir.message", ex.getLocalizedMessage())); } } } @@ -145,7 +145,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { // Check that we have roughly enough disk space left to complete the operation long freeDiskSpace = IngestServices.getInstance().getFreeDiskSpace(); if ((file.getSize() * 2) > freeDiskSpace) { - logger.log(Level.SEVERE, "PhotoRec error processing {0} with {1} Not enough space on primary disk to carve unallocated space.", + logger.log(Level.SEVERE, "PhotoRec error processing {0} with {1} Not enough space on primary disk to carve unallocated space.", // NON-NLS new Object[]{file.getName(), PhotoRecCarverIngestModuleFactory.getModuleName()}); // NON-NLS return IngestModule.ProcessResult.ERROR; } @@ -163,9 +163,9 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { // Scan the file with Unallocated Carver. ProcessBuilder processAndSettings = new ProcessBuilder( "\"" + executableFile + "\"", - "/d", + "/d", // NON-NLS "\"" + outputDirPath.toAbsolutePath() + File.separator + PHOTOREC_RESULTS_BASE + "\"", - "/cmd", + "/cmd", // NON-NLS "\"" + tempFilePath.toFile() + "\"", "search"); // NON_NLS @@ -175,7 +175,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { processAndSettings.redirectOutput(Redirect.appendTo(log)); int exitValue = ExecUtil.execute(processAndSettings, new FileIngestModuleProcessTerminator(this.context)); - + if (this.context.fileIngestIsCancelled() == true) { // if it was cancelled by the user, result is OK // cleanup the output path @@ -194,7 +194,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { if (null != tempFilePath && Files.exists(tempFilePath)) { tempFilePath.toFile().delete(); } - logger.log(Level.SEVERE, "PhotoRec carver returned error exit value = {0} when scanning {1}", + logger.log(Level.SEVERE, "PhotoRec carver returned error exit value = {0} when scanning {1}", // NON-NLS new Object[]{exitValue, file.getName()}); // NON-NLS return IngestModule.ProcessResult.ERROR; } @@ -205,13 +205,14 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { oldAuditFile.renameTo(newAuditFile); Path pathToRemove = Paths.get(outputDirPath.toAbsolutePath().toString()); - DirectoryStream stream = Files.newDirectoryStream(pathToRemove); - for (Path entry : stream) { - if (Files.isDirectory(entry)) { - FileUtil.deleteDir(new File(entry.toString())); + try (DirectoryStream stream = Files.newDirectoryStream(pathToRemove)) { + for (Path entry : stream) { + if (Files.isDirectory(entry)) { + FileUtil.deleteDir(new File(entry.toString())); + } } } - + // Now that we've cleaned up the folders and data files, parse the xml output file to add carved items into the database PhotoRecCarverOutputParser parser = new PhotoRecCarverOutputParser(outputDirPath); List theList = parser.parse(newAuditFile, id, file); diff --git a/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java b/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java index 5555b0ce2c..68851e9caf 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java @@ -62,7 +62,7 @@ public enum MiscTypes implements EventType, ArtifactEventType { final BlackboardAttribute longEnd = attrMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE_END); return String.format("from %1$g %2$g to %3$g %4$g", latStart.getValueDouble(), longStart.getValueDouble(), latEnd.getValueDouble(), longEnd.getValueDouble()); }), - GPS_TRACKPOINT("Location History", "gps_trackpoint.png", + GPS_TRACKPOINT("Location History", "gps-trackpoint.png", BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, new AttributeExtractor(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME), diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index b808dd0d96..28aa07b92a 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 17:10:10 -0400 +#Wed, 17 Dec 2014 13:11:49 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index f0109bc92b..3a5c88bbe9 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 12:13:36 -0400 +#Wed, 17 Dec 2014 13:11:49 -0500 CTL_MainWindow_Title=Autopsy 3.1.1 CTL_MainWindow_Title_No_Project=Autopsy 3.1.1 diff --git a/docs/doxygen/modReport.dox b/docs/doxygen/modReport.dox index c61471f5f1..17d7006fff 100644 --- a/docs/doxygen/modReport.dox +++ b/docs/doxygen/modReport.dox @@ -54,6 +54,7 @@ If you implement FileReportModule, the overriden methods will be: As when generating table module reports, Autopsy will iterate through a list of user selected data (which are represented by FileReportDataTypes), and call addRow(AbstractFile toAdd, List columns) for every abstract file in the case. Developers are guaranteed that the order of method calls will be startReport(), startTable(List headers), addRow(AbstractFile toAdd, List columns), AbstractFile toAdd, List columns),..., endTable(), endReport(). \subsection report_create_module_general Creating a General Report Module + If you implement GeneralReportModule, the overriden methods will be: - org.sleuthkit.autopsy.report.GeneralReportModule.generateReport(String reportPath, ReportProgressPanel progressPanel) - org.sleuthkit.autopsy.report.GeneralReportModule.getConfigurationPanel() @@ -62,7 +63,8 @@ For general report modules, Autopsy will simply call the generateReport(String r When updating the progress panel, it is recommended to update it as infrequently as possible, while still keeping the user informed. If your report processes 100,000 files and you chose to update the UI each time a file is reviewed, the UI would freeze when trying to process all your requests. This would cause problems to not only your reporting module, but to other modules running in parallel. A safer approach would be to update the UI every 1,000 files, or when a certain "category" of the files being processed has changed. For example, the HTML report module increments the progress bar and changes the processing label every time a new Blackboard Artifact Type is being processed. -Autopsy will also display the panel returned by getConfigurationPanel() in the generate report wizard. This panel can be used to allow the user custom controls over the report. +Autopsy will also display the panel returned by getConfigurationPanel() in the generate report wizard. This panel can be used to allow the user custom controls over the report. To make this panel, use NetBeans to make a new JPanel class and use its layout interface to put the UI widgets in the places that you want them. Then, your getConfigurationPanel() method should create an instance of that class and return it. + Typically a general report module should interact with both the Blackboard API in the org.sleuthkit.datamodel.SleuthkitCase class, in addition to an API (possibly external/thirdparty) to convert Blackboard Artifacts to the desired reporting format.