From 4658dac1c361d9be46246e26f1c058d273c75647 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 21 Nov 2014 00:36:12 -0500 Subject: [PATCH 01/84] Add class skeletons and new method skeletons for user-defined file type sigs --- .../filetypeid/FileTypeIdIngestModule.java | 226 ++++++++++-------- .../filetypeid/FileTypeIdModuleFactory.java | 55 ++++- .../filetypeid/FileTypeIdModuleSettings.java | 4 + .../FileTypeIdModuleSettingsPanel.java | 15 +- .../FileTypeIdOptionsPanelController.java | 96 ++++++++ .../filetypeid/FileTypeIdSettingsPanel.form | 28 +++ .../filetypeid/FileTypeIdSettingsPanel.java | 93 +++++++ 7 files changed, 410 insertions(+), 107 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 2abb69974a..df575fe18d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -27,9 +27,6 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; @@ -45,102 +42,11 @@ public class FileTypeIdIngestModule implements FileIngestModule { private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); private static final long MIN_FILE_SIZE = 512; private final FileTypeIdModuleSettings settings; - private long jobId; - + private long jobId; private static final HashMap totalsForIngestJobs = new HashMap<>(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); - private TikaFileTypeDetector tikaDetector = new TikaFileTypeDetector(); + private final TikaFileTypeDetector tikaDetector = new TikaFileTypeDetector(); - private static class IngestJobTotals { - long matchTime = 0; - long numFiles = 0; - } - - /** - * Update the match time total and increment num of files for this job - * @param ingestJobId - * @param matchTimeInc amount of time to add - */ - private static synchronized void addToTotals(long ingestJobId, long matchTimeInc) { - IngestJobTotals ingestJobTotals = totalsForIngestJobs.get(ingestJobId); - if (ingestJobTotals == null) { - ingestJobTotals = new IngestJobTotals(); - totalsForIngestJobs.put(ingestJobId, ingestJobTotals); - } - - ingestJobTotals.matchTime += matchTimeInc; - ingestJobTotals.numFiles++; - totalsForIngestJobs.put(ingestJobId, ingestJobTotals); - } - - FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { - this.settings = settings; - } - - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - jobId = context.getJobId(); - refCounter.incrementAndGet(jobId); - } - - @Override - public ProcessResult process(AbstractFile abstractFile) { - // skip non-files - if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) - || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { - - return ProcessResult.OK; - } - - if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) { - return ProcessResult.OK; - } - - if (abstractFile.getSize() < MIN_FILE_SIZE) { - return ProcessResult.OK; - } - - try { - long startTime = System.currentTimeMillis(); - tikaDetector.detectAndSave(abstractFile); - addToTotals(jobId, (System.currentTimeMillis() - startTime)); //add match time - return ProcessResult.OK; - } catch (TskException ex) { - logger.log(Level.WARNING, "Error matching file signature", ex); //NON-NLS - return ProcessResult.ERROR; - } catch (Exception e) { - logger.log(Level.WARNING, "Error matching file signature", e); //NON-NLS - return ProcessResult.ERROR; - } - } - - @Override - public void shutDown() { - // We only need to post the summary msg from the last module per job - if (refCounter.decrementAndGet(jobId) == 0) { - IngestJobTotals jobTotals; - synchronized(this) { - jobTotals = totalsForIngestJobs.remove(jobId); - } - if (jobTotals != null) { - StringBuilder detailsSb = new StringBuilder(); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - detailsSb.append("\n"); //NON-NLS - detailsSb.append("\n"); //NON-NLS - detailsSb.append("
").append(FileTypeIdModuleFactory.getModuleName()).append("
") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) - .append("").append(jobTotals.matchTime).append("
") //NON-NLS - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) - .append("").append(jobTotals.numFiles).append("
"); //NON-NLS - IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "FileTypeIdIngestModule.complete.srvMsg.text"), - detailsSb.toString())); - } - } - } - /** * Validate if a given mime type is in the detector's registry. * @@ -157,4 +63,130 @@ public class FileTypeIdIngestModule implements FileIngestModule { TikaFileTypeDetector detector = new TikaFileTypeDetector(); return detector.isMimeTypeDetectable(mimeType); } -} \ No newline at end of file + + /** + * Creates an ingest module that detects the type of a file based on + * signature (magic) values. Posts results to the blackboard. + * + * @param settings The ingest module settings. + */ + FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { + this.settings = settings; + } + + /** + * @inheritDoc + */ + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + refCounter.incrementAndGet(jobId); + } + + /** + * @inheritDoc + */ + @Override + public ProcessResult process(AbstractFile abstractFile) { + /** + * Skip unallocated space and unused blocks files. + */ + if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { + + return ProcessResult.OK; + } + + /** + * Skip known files if configured to do so. + */ + if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) { + return ProcessResult.OK; + } + + /** + * Filter out very small files to minimize false positives. + */ + // RJCTODO: Make this size a setting + if (abstractFile.getSize() < MIN_FILE_SIZE) { + return ProcessResult.OK; + } + + try { + long startTime = System.currentTimeMillis(); + + // RJCTODO: Add code here to use nay user-defined signstures first; + // if there is a match, post a intersting file hit, if the user has + // elected alerts + tikaDetector.detectAndSave(abstractFile); + + addToTotals(jobId, (System.currentTimeMillis() - startTime)); + return ProcessResult.OK; + } catch (TskException ex) { + logger.log(Level.WARNING, "Error matching file signature", ex); //NON-NLS + return ProcessResult.ERROR; + } catch (Exception e) { + logger.log(Level.WARNING, "Error matching file signature", e); //NON-NLS + return ProcessResult.ERROR; + } + } + + /** + * @inheritDoc + */ + @Override + public void shutDown() { + /** + * If this is the instance of this module for this ingest job, post a + * summary message to the ingest messages box. + */ + if (refCounter.decrementAndGet(jobId) == 0) { + IngestJobTotals jobTotals; + synchronized (this) { + jobTotals = totalsForIngestJobs.remove(jobId); + } + if (jobTotals != null) { + StringBuilder detailsSb = new StringBuilder(); + detailsSb.append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + detailsSb.append("\n"); //NON-NLS + detailsSb.append("\n"); //NON-NLS + detailsSb.append("
").append(FileTypeIdModuleFactory.getModuleName()).append("
") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) + .append("").append(jobTotals.matchTime).append("
") //NON-NLS + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) + .append("").append(jobTotals.numFiles).append("
"); //NON-NLS + IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileTypeIdIngestModule.complete.srvMsg.text"), + detailsSb.toString())); + } + } + } + + /** + * Update the match time total and increment number of files processed for + * this ingest job. + * + * @param jobId The ingest job identifier. + * @param matchTimeInc Amount of time to add. + */ + private static synchronized void addToTotals(long jobId, long matchTimeInc) { + IngestJobTotals ingestJobTotals = totalsForIngestJobs.get(jobId); + if (ingestJobTotals == null) { + ingestJobTotals = new IngestJobTotals(); + totalsForIngestJobs.put(jobId, ingestJobTotals); + } + + ingestJobTotals.matchTime += matchTimeInc; + ingestJobTotals.numFiles++; + totalsForIngestJobs.put(jobId, ingestJobTotals); + } + + private static class IngestJobTotals { + + long matchTime = 0; + long numFiles = 0; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java index 6688fd8027..73d1e140e5 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java @@ -24,69 +24,114 @@ import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleFactory; import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; /** - * An factory that creates file ingest modules that determine the types of - * files. + * A factory that creates file ingest modules that determine the types of files. */ @ServiceProvider(service = IngestModuleFactory.class) public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { + /** + * @inheritDoc + */ @Override public String getModuleDisplayName() { return getModuleName(); } + /** + * Gets the module display name. + * + * @return The name string. + */ static String getModuleName() { return NbBundle.getMessage(FileTypeIdIngestModule.class, "FileTypeIdIngestModule.moduleName.text"); } + /** + * @inheritDoc + */ @Override public String getModuleDescription() { return NbBundle.getMessage(FileTypeIdIngestModule.class, "FileTypeIdIngestModule.moduleDesc.text"); } + /** + * @inheritDoc + */ @Override public String getModuleVersionNumber() { return Version.getVersion(); } + /** + * @inheritDoc + */ + @Override + public boolean hasGlobalSettingsPanel() { + return true; + } + + /** + * @inheritDoc + */ + @Override + public IngestModuleGlobalSettingsPanel getGlobalSettingsPanel() { + return new FileTypeIdSettingsPanel(); + } + + /** + * @inheritDoc + */ @Override public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { return new FileTypeIdModuleSettings(); } + /** + * @inheritDoc + */ @Override public boolean hasIngestJobSettingsPanel() { return true; } + /** + * @inheritDoc + */ @Override public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { assert settings instanceof FileTypeIdModuleSettings; if (!(settings instanceof FileTypeIdModuleSettings)) { throw new IllegalArgumentException(NbBundle.getMessage(this.getClass(), - "FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg")); - } + "FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg")); + } return new FileTypeIdModuleSettingsPanel((FileTypeIdModuleSettings) settings); } + /** + * @inheritDoc + */ @Override public boolean isFileIngestModuleFactory() { return true; } + /** + * @inheritDoc + */ @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { assert settings instanceof FileTypeIdModuleSettings; if (!(settings instanceof FileTypeIdModuleSettings)) { throw new IllegalArgumentException( NbBundle.getMessage(this.getClass(), "FileTypeIdModuleFactory.createFileIngestModule.exception.msg")); - } + } return new FileTypeIdIngestModule((FileTypeIdModuleSettings) settings); } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java index c472596be4..31e077a875 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java @@ -23,6 +23,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; /** * Ingest job options for the file type identifier ingest module instances. */ +// @Deprecated This class is not for public use public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { private static final long serialVersionUID = 1L; @@ -35,6 +36,9 @@ public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { this.skipKnownFiles = skipKnownFiles; } + /** + * @inheritDoc + */ @Override public long getVersionNumber() { return serialVersionUID; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java index 3839d2c711..bf86217a50 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java @@ -29,20 +29,25 @@ final class FileTypeIdModuleSettingsPanel extends IngestModuleIngestJobSettingsP private final FileTypeIdModuleSettings settings; - public FileTypeIdModuleSettingsPanel(FileTypeIdModuleSettings settings) { + // NOTE: This was declared public, but was inaccessible because the class is + // not public + FileTypeIdModuleSettingsPanel(FileTypeIdModuleSettings settings) { this.settings = settings; initComponents(); customizeComponents(); } - private void customizeComponents() { - skipKnownCheckBox.setSelected(settings.skipKnownFiles()); - } - + /** + * @inheritDoc + */ @Override public IngestModuleIngestJobSettings getSettings() { return settings; } + + private void customizeComponents() { + skipKnownCheckBox.setSelected(settings.skipKnownFiles()); + } /** * This method is called from within the constructor to initialize the form. diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java new file mode 100644 index 0000000000..4209dd9dc5 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java @@ -0,0 +1,96 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.beans.PropertyChangeListener; +import java.beans.PropertyChangeSupport; +import javax.swing.JComponent; +import javax.swing.SwingUtilities; +import org.netbeans.spi.options.OptionsPanelController; +import org.openide.util.HelpCtx; +import org.openide.util.Lookup; + +@OptionsPanelController.TopLevelRegistration( + categoryName = "#OptionsCategory_Name_FileTypeId", + iconBase = "org/sleuthkit/autopsy/corecomponents/display-options.png", + keywords = "#OptionsCategory_Keywords_FileTypeId", + keywordsCategory = "FileTypeId" +) +@org.openide.util.NbBundle.Messages({"OptionsCategory_Name_FileTypeId=FileTypeId", "OptionsCategory_Keywords_FileTypeId=FileTypeId"}) +public final class FileTypeIdOptionsPanelController extends OptionsPanelController { + + private FileTypeIdSettingsPanel panel; + private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); + private boolean changed; + + @Override + public void update() { + getPanel().load(); + changed = false; + } + + @Override + public void applyChanges() { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + getPanel().store(); + changed = false; + } + }); + } + + @Override + public void cancel() { + // need not do anything special, if no changes have been persisted yet + } + + @Override + public boolean isValid() { + return getPanel().valid(); + } + + @Override + public boolean isChanged() { + return changed; + } + + @Override + public HelpCtx getHelpCtx() { + return null; // new HelpCtx("...ID") if you have a help set + } + + @Override + public JComponent getComponent(Lookup masterLookup) { + return getPanel(); + } + + @Override + public void addPropertyChangeListener(PropertyChangeListener l) { + pcs.addPropertyChangeListener(l); + } + + @Override + public void removePropertyChangeListener(PropertyChangeListener l) { + pcs.removePropertyChangeListener(l); + } + + private FileTypeIdSettingsPanel getPanel() { + if (panel == null) { + panel = new FileTypeIdSettingsPanel(); + } + return panel; + } + + void changed() { + if (!changed) { + changed = true; + pcs.firePropertyChange(OptionsPanelController.PROP_CHANGED, false, true); + } + pcs.firePropertyChange(OptionsPanelController.PROP_VALID, null, null); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form new file mode 100644 index 0000000000..4f9abb50dc --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form @@ -0,0 +1,28 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java new file mode 100644 index 0000000000..10a0291cfe --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java @@ -0,0 +1,93 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; + +/** + * RJCTODO + */ +class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { + + /** + * Creates... RJCTODO + */ + FileTypeIdSettingsPanel() { + initComponents(); + } + + /** + * @inheritDoc + */ + @Override + public void saveSettings() { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + void load() { + // RJCTODO read settings and initialize GUI + // Example: + // someCheckBox.setSelected(Preferences.userNodeForPackage(FileTypeIdPanel.class).getBoolean("someFlag", false)); + // or for org.openide.util with API spec. version >= 7.4: + // someCheckBox.setSelected(NbPreferences.forModule(FileTypeIdPanel.class).getBoolean("someFlag", false)); + // or: + // someTextField.setText(SomeSystemOption.getDefault().getSomeStringProperty()); + } + + void store() { + // RJCTODO store modified settings + // Example: + // Preferences.userNodeForPackage(FileTypeIdPanel.class).putBoolean("someFlag", someCheckBox.isSelected()); + // or for org.openide.util with API spec. version >= 7.4: + // NbPreferences.forModule(FileTypeIdPanel.class).putBoolean("someFlag", someCheckBox.isSelected()); + // or: + // SomeSystemOption.getDefault().setSomeStringProperty(someTextField.getText()); + } + + boolean valid() { + // RJCTODO check whether form is consistent and complete + return true; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGap(0, 400, Short.MAX_VALUE) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGap(0, 300, Short.MAX_VALUE) + ); + }// //GEN-END:initComponents + + + // Variables declaration - do not modify//GEN-BEGIN:variables + // End of variables declaration//GEN-END:variables + +} From 60f33f61bb3bb320e3b9218e29a7e31e1d12c229 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 21 Nov 2014 07:57:04 -0500 Subject: [PATCH 02/84] Check in partial implementation of user defined file type sigs --- .../UserDefinedFileTypeDetector.java | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java new file mode 100644 index 0000000000..2adcada8cf --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java @@ -0,0 +1,171 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * RJCTODO + */ +class UserDefinedFileTypeDetector { + + private final List matchers; + private String detectedType; + + static UserDefinedFileTypeDetector createDetector(String sigFilePath) { + UserDefinedFileTypeDetector detector = new UserDefinedFileTypeDetector(); + detector.loadSignatures(sigFilePath); + return detector; + } + + /** + * RJCTODO + * + * @param sigFilePath + */ + UserDefinedFileTypeDetector() { + this.matchers = new ArrayList<>(); + } + + /** + * RJCTODO + */ + private void loadSignatures(String sigFilePath) { + // RJCTODO: Load signature file, creating + } + + /** + * + * @param file + * @return + */ + boolean detect(AbstractFile file) { + for (SignatureMatcher matcher : this.matchers) { + if (matcher.matched(file)) { + this.detectedType = matcher.getMatchType(); + return true; + } + } + this.detectedType = ""; // RJCTODO: Wrap this stuff in a class + return false; + } + + /** + * RJCTODO + * @return + */ + String getDetectedType() { + // RJCTODO + return ""; + } + + /** + * + */ + private static class SignatureMatcher { + + private final List matchers; + private final String matchType; + + /** + * RJCTODO + * + * @param matchType + */ + SignatureMatcher(String matchType) { + this.matchType = matchType; + this.matchers = new ArrayList<>(); + } + + /** + * RJCTODO + * + * @param file + * @return + */ + boolean matched(AbstractFile file) { + for (Matcher matcher : this.matchers) { + if (!matcher.matched(file)) { + return false; + } + } + return true; + } + + /** + * RJCTODO + * + * @return + */ + String getMatchType() { + return this.matchType; + } + + } + + private static interface Matcher { + + /** + * RJCTODO + * + * @param file + * @return + */ + boolean matched(AbstractFile file); + + } + + /** + * A signature matcher that looks for a sequence of bytes at a specified + * offset. + */ + private static class ByteMatcher implements Matcher { + + private final long offset; + private final byte[] signature; + private final byte[] buffer; + + private ByteMatcher(long offset, byte[] signature) { + this.offset = offset; + this.signature = signature; + this.buffer = new byte[signature.length]; + } + + /** + * @inheritDoc + */ + @Override + public boolean matched(AbstractFile file) { + try { + // RJCTODO: Confirm this logic + int bytesRead = file.read(this.buffer, offset, buffer.length); + return bytesRead == buffer.length ? Arrays.equals(this.buffer, this.signature) : false; + } catch (TskCoreException ex) { + // RJCTODO + return false; + } + } + + } + +} From cfc58077979dfe5d393202c4bb00cac0bb6078f0 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 21 Nov 2014 17:46:07 -0500 Subject: [PATCH 03/84] Check in incomplete editing of user defined file sigs feature --- .../modules/filetypeid/Bundle.properties | 12 + .../filetypeid/FileTypeIdSettingsPanel.form | 261 +++++++++++++++++- .../filetypeid/FileTypeIdSettingsPanel.java | 167 ++++++++++- .../UserDefinedFileTypeDetector.java | 171 ------------ .../UserDefinedFileTypeIdentifier.java | 81 ++++++ .../filetypeid/UserDefinedFileTypes.java | 173 ++++++++++++ 6 files changed, 687 insertions(+), 178 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 1369d3d332..7ac88eec61 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -8,3 +8,15 @@ FileTypeIdIngestModule.complete.srvMsg.text=File Type Id Results FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings FileTypeIdModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) +FileTypeIdSettingsPanel.jTextField2.text= +FileTypeIdSettingsPanel.jButton1.text=New Type +FileTypeIdSettingsPanel.jButton2.text=DeleteType +FileTypeIdSettingsPanel.jButton3.text=Save Type +FileTypeIdSettingsPanel.mimeTypeTextField.text= +FileTypeIdSettingsPanel.mimeTypeLabel.text=Mime Type +FileTypeIdSettingsPanel.signatureTypeLabel.text=Signature Type +FileTypeIdSettingsPanel.signatureLabel.text=Signature +FileTypeIdSettingsPanel.offsetLabel.text=Offset +FileTypeIdSettingsPanel.offsetTextField.text= +FileTypeIdSettingsPanel.hexPrefixLabel.text=0x +FileTypeIdSettingsPanel.jTextArea1.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0. diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form index 4f9abb50dc..27806b6718 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form @@ -1,6 +1,10 @@ -
+ + + + + @@ -16,13 +20,264 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java index 10a0291cfe..8e5018d3e8 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java @@ -30,6 +30,7 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { */ FileTypeIdSettingsPanel() { initComponents(); + } /** @@ -39,7 +40,7 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { public void saveSettings() { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } - + void load() { // RJCTODO read settings and initialize GUI // Example: @@ -61,7 +62,6 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { } boolean valid() { - // RJCTODO check whether form is consistent and complete return true; } @@ -74,20 +74,179 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { // //GEN-BEGIN:initComponents private void initComponents() { + buttonGroup1 = new javax.swing.ButtonGroup(); + jScrollPane1 = new javax.swing.JScrollPane(); + jList1 = new javax.swing.JList(); + jSeparator1 = new javax.swing.JSeparator(); + mimeTypeLabel = new javax.swing.JLabel(); + mimeTypeTextField = new javax.swing.JTextField(); + signatureTypeLabel = new javax.swing.JLabel(); + jTextField2 = new javax.swing.JTextField(); + offsetLabel = new javax.swing.JLabel(); + offsetTextField = new javax.swing.JTextField(); + jButton1 = new javax.swing.JButton(); + jButton2 = new javax.swing.JButton(); + jButton3 = new javax.swing.JButton(); + hexPrefixLabel = new javax.swing.JLabel(); + signatureTypeComboBox = new javax.swing.JComboBox(); + signatureLabel = new javax.swing.JLabel(); + jScrollPane2 = new javax.swing.JScrollPane(); + jTextArea1 = new javax.swing.JTextArea(); + + jScrollPane1.setViewportView(jList1); + + jSeparator1.setOrientation(javax.swing.SwingConstants.VERTICAL); + + org.openide.awt.Mnemonics.setLocalizedText(mimeTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.mimeTypeLabel.text")); // NOI18N + + mimeTypeTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.mimeTypeTextField.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(signatureTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.signatureTypeLabel.text")); // NOI18N + + jTextField2.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jTextField2.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(offsetLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.offsetLabel.text")); // NOI18N + + offsetTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.offsetTextField.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(jButton1, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jButton1.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(jButton2, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jButton2.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(jButton3, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jButton3.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(hexPrefixLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.hexPrefixLabel.text")); // NOI18N + + signatureTypeComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "Bytes (Hex)", "String", " " })); + + org.openide.awt.Mnemonics.setLocalizedText(signatureLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.signatureLabel.text")); // NOI18N + + jTextArea1.setEditable(false); + jTextArea1.setColumns(20); + jTextArea1.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N + jTextArea1.setLineWrap(true); + jTextArea1.setRows(5); + jTextArea1.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jTextArea1.text")); // NOI18N + jTextArea1.setWrapStyleWord(true); + jScrollPane2.setViewportView(jTextArea1); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGap(0, 400, Short.MAX_VALUE) + .addGroup(layout.createSequentialGroup() + .addGap(26, 26, 26) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(10, 10, 10) + .addComponent(jButton1) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(jButton2)) + .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(37, 37, 37) + .addComponent(jButton3) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGroup(layout.createSequentialGroup() + .addGap(18, 18, 18) + .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) + .addGroup(layout.createSequentialGroup() + .addComponent(signatureTypeLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 82, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGap(0, 0, Short.MAX_VALUE)) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(signatureLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(hexPrefixLabel) + .addGap(2, 2, 2)) + .addGroup(layout.createSequentialGroup() + .addComponent(offsetLabel) + .addGap(44, 44, 44))) + .addGap(5, 5, 5) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 84, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(jTextField2, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))))) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGap(6, 6, 6) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) + .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(layout.createSequentialGroup() + .addComponent(mimeTypeLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 181, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addGap(0, 0, Short.MAX_VALUE))) + .addContainerGap()))) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGap(0, 300, Short.MAX_VALUE) + .addGroup(layout.createSequentialGroup() + .addGap(16, 16, 16) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) + .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(layout.createSequentialGroup() + .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(mimeTypeLabel) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(signatureTypeLabel) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(jTextField2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(hexPrefixLabel) + .addComponent(signatureLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(offsetLabel)) + .addGap(12, 12, 12))) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) + .addGroup(layout.createSequentialGroup() + .addGap(18, 18, 18) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(jButton1) + .addComponent(jButton2))) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(jButton3)))) + .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.ButtonGroup buttonGroup1; + private javax.swing.JLabel hexPrefixLabel; + private javax.swing.JButton jButton1; + private javax.swing.JButton jButton2; + private javax.swing.JButton jButton3; + private javax.swing.JList jList1; + private javax.swing.JScrollPane jScrollPane1; + private javax.swing.JScrollPane jScrollPane2; + private javax.swing.JSeparator jSeparator1; + private javax.swing.JTextArea jTextArea1; + private javax.swing.JTextField jTextField2; + private javax.swing.JLabel mimeTypeLabel; + private javax.swing.JTextField mimeTypeTextField; + private javax.swing.JLabel offsetLabel; + private javax.swing.JTextField offsetTextField; + private javax.swing.JLabel signatureLabel; + private javax.swing.JComboBox signatureTypeComboBox; + private javax.swing.JLabel signatureTypeLabel; // End of variables declaration//GEN-END:variables } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java deleted file mode 100644 index 2adcada8cf..0000000000 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.modules.filetypeid; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; - -/** - * RJCTODO - */ -class UserDefinedFileTypeDetector { - - private final List matchers; - private String detectedType; - - static UserDefinedFileTypeDetector createDetector(String sigFilePath) { - UserDefinedFileTypeDetector detector = new UserDefinedFileTypeDetector(); - detector.loadSignatures(sigFilePath); - return detector; - } - - /** - * RJCTODO - * - * @param sigFilePath - */ - UserDefinedFileTypeDetector() { - this.matchers = new ArrayList<>(); - } - - /** - * RJCTODO - */ - private void loadSignatures(String sigFilePath) { - // RJCTODO: Load signature file, creating - } - - /** - * - * @param file - * @return - */ - boolean detect(AbstractFile file) { - for (SignatureMatcher matcher : this.matchers) { - if (matcher.matched(file)) { - this.detectedType = matcher.getMatchType(); - return true; - } - } - this.detectedType = ""; // RJCTODO: Wrap this stuff in a class - return false; - } - - /** - * RJCTODO - * @return - */ - String getDetectedType() { - // RJCTODO - return ""; - } - - /** - * - */ - private static class SignatureMatcher { - - private final List matchers; - private final String matchType; - - /** - * RJCTODO - * - * @param matchType - */ - SignatureMatcher(String matchType) { - this.matchType = matchType; - this.matchers = new ArrayList<>(); - } - - /** - * RJCTODO - * - * @param file - * @return - */ - boolean matched(AbstractFile file) { - for (Matcher matcher : this.matchers) { - if (!matcher.matched(file)) { - return false; - } - } - return true; - } - - /** - * RJCTODO - * - * @return - */ - String getMatchType() { - return this.matchType; - } - - } - - private static interface Matcher { - - /** - * RJCTODO - * - * @param file - * @return - */ - boolean matched(AbstractFile file); - - } - - /** - * A signature matcher that looks for a sequence of bytes at a specified - * offset. - */ - private static class ByteMatcher implements Matcher { - - private final long offset; - private final byte[] signature; - private final byte[] buffer; - - private ByteMatcher(long offset, byte[] signature) { - this.offset = offset; - this.signature = signature; - this.buffer = new byte[signature.length]; - } - - /** - * @inheritDoc - */ - @Override - public boolean matched(AbstractFile file) { - try { - // RJCTODO: Confirm this logic - int bytesRead = file.read(this.buffer, offset, buffer.length); - return bytesRead == buffer.length ? Arrays.equals(this.buffer, this.signature) : false; - } catch (TskCoreException ex) { - // RJCTODO - return false; - } - } - - } - -} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java new file mode 100644 index 0000000000..d96d7b4f5f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -0,0 +1,81 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.util.ArrayList; +import java.util.List; +import org.sleuthkit.datamodel.AbstractFile; + +/** + * Does file type identification with user-defined file type signatures. + */ +class UserDefinedFileTypeIdentifier { + + private static final String USER_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; + private static final String AUTOPSY_DEFINITIONS_FILE = "AutopsyFileTypeDefinitions.xml"; + private final List signatures; + + /** + * Creates an object that does file type identification with user-defined + * file type signatures. + * + * @param sigFilePath A path to a signature definitions file. + */ + static UserDefinedFileTypeIdentifier createDetector(String sigFilePath) { + UserDefinedFileTypeIdentifier detector = new UserDefinedFileTypeIdentifier(); + detector.loadSignatures(sigFilePath); + return detector; + } + + /** + * Create an object that does file type identification with user-defined + * file type signatures. + */ + private UserDefinedFileTypeIdentifier() { + this.signatures = new ArrayList<>(); + } + + /** + * Loads a set of user-defined file type signatures from a file. + * + * @param sigFilePath The path to the signature definitions file. + */ + private void loadSignatures(String sigFilePath) { + // RJCTODO: Load signature file, creating + } + + /** + * Attempts to identify a file using the set of user-defined file type + * signatures. + * + * @param file The file to type. + * @return A MIME type string or the empty string if identification fails. + */ + String identify(AbstractFile file) { + String type = ""; + for (UserDefinedFileTypes.FileSignature signature : this.signatures) { + if (signature.containedIn(file)) { + type = signature.getMimeType(); + break; + } + } + return type; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java new file mode 100644 index 0000000000..fcd275a89f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java @@ -0,0 +1,173 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.sleuthkit.autopsy.externalresults.ExternalResultsXMLParser; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +/** + * RJCTODO + */ +public class UserDefinedFileTypes { + + private static final String ROOT_FILE_TYPES_ELEMENT = "fileTypes"; // NON-NLS + private static final String FILE_TYPE_ELEMENT = "fileType"; // NON-NLS + private static final String MIME_TYPE_ELEMENT = "mimeType"; // NON-NLS + private static final String SIGNATURE_ELEMENT = "signature"; // NON-NLS + private static final String OFFSET_ELEMENT = "offset"; + + /** + * An association between a MIME type and a signature within a file. + */ + static interface FileSignature { + + /** + * Gets the MIME type associated with this signature. + * + * @return The MIME type string. + */ + String getMimeType(); + + /** + * Determines whether or not a file contains the signature. + * + * @param file The file to test. + * @return True or false. + */ + boolean containedIn(AbstractFile file); + + } + + /** + * An association between a MIME type and a byte signature at a specified + * offset within a file. + */ + static class ByteSignature implements FileSignature { + + private final String mimeType; + private final byte[] signatureBytes; + private final long offset; + private final byte[] buffer; + + /** + * Creates an association between a MIME type and a byte signature at a + * specified offset. + * + * @param mimeType The MIME type string. + * @param signatureBytes The bytes of the signature. + * @param offset The offset of the signature within a file. + */ + private ByteSignature(String mimeType, byte[] signatureBytes, long offset) { + this.mimeType = mimeType; + this.signatureBytes = signatureBytes; + this.offset = offset; + this.buffer = new byte[signatureBytes.length]; + } + + /** + * @inheritDoc + */ + @Override + public String getMimeType() { + return this.mimeType; + } + + /** + * @inheritDoc + */ + @Override + public boolean containedIn(AbstractFile file) { + try { + int bytesRead = file.read(this.buffer, offset, buffer.length); + return bytesRead == buffer.length ? Arrays.equals(this.buffer, this.signatureBytes) : false; + } catch (TskCoreException ex) { + return false; + } + } + + } + + /** + * Provides a mechanism for persisting user-defined file types. + */ + static class Writer { + + /** + * Reads user-defined file type definitions from a file. + * + * @param signatures A collection of file signatures. + * @param filePath The path to the file. + */ + static void writeFileTypes(List signatures, String filePath) { + + } + + } + + /** + * Provides a method for reading persisted user-defined file types. + */ + static class Reader { + + /** + * Reads user-defined file type definitions from a file. + * + * @param filePath The path to the file. + * @return A collection of file signatures. + */ + static List readFileTypes(String filePath) { + // RJCTODO: + try { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document doc = builder.parse(new FileInputStream(filePath)); + final Document doc = XMLUtil.loadDoc(ExternalResultsXMLParser.class, this.resultsFilePath, XSD_FILE); + if (doc != null) { + final Element rootElem = doc.getDocumentElement(); + if (rootElem != null && rootElem.getNodeName().equals(ExternalResultsXMLParser.TagNames.ROOT_ELEM.toString())) { + + } catch (ParserConfigurationException e) { + } catch (SAXException e) { + } catch (IOException e) { + } + + + + + return Collections.emptyList(); + } + + } + +} From 2ace5b2fbbe152447a3d03278d78906df2c4d33c Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 24 Nov 2014 16:58:59 -0500 Subject: [PATCH 04/84] Checdk in work on user defined file types --- .../modules/filetypeid/Bundle.properties | 3 +- .../filetypeid/FileTypeIdSettingsPanel.form | 47 ++-- .../filetypeid/FileTypeIdSettingsPanel.java | 40 ++-- .../UserDefinedFileTypeIdentifier.java | 4 +- .../filetypeid/UserDefinedFileTypes.java | 220 +++++++++++++++--- 5 files changed, 254 insertions(+), 60 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 7ac88eec61..58978a9b5d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -19,4 +19,5 @@ FileTypeIdSettingsPanel.signatureLabel.text=Signature FileTypeIdSettingsPanel.offsetLabel.text=Offset FileTypeIdSettingsPanel.offsetTextField.text= FileTypeIdSettingsPanel.hexPrefixLabel.text=0x -FileTypeIdSettingsPanel.jTextArea1.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0. +FileTypeIdSettingsPanel.jTextArea1.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. +FileTypeIdSettingsPanel.postHitCheckBox.text=Post interesting file hit when found diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form index 27806b6718..efb8311caf 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form @@ -40,25 +40,25 @@ - + - + - + - + - + @@ -69,18 +69,25 @@ - - - - - - - + + + + + + + + + + + + + + @@ -96,11 +103,11 @@ - + - + - + @@ -121,7 +128,8 @@ - + + @@ -279,5 +287,12 @@ + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java index 8e5018d3e8..c0ffdbfd3b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java @@ -18,7 +18,9 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.util.List; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; +import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypes.FileTypeSignature; /** * RJCTODO @@ -42,6 +44,8 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { } void load() { + List fileTypeSignatures; + // RJCTODO read settings and initialize GUI // Example: // someCheckBox.setSelected(Preferences.userNodeForPackage(FileTypeIdPanel.class).getBoolean("someFlag", false)); @@ -92,6 +96,7 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { signatureLabel = new javax.swing.JLabel(); jScrollPane2 = new javax.swing.JScrollPane(); jTextArea1 = new javax.swing.JTextArea(); + postHitCheckBox = new javax.swing.JCheckBox(); jScrollPane1.setViewportView(jList1); @@ -130,6 +135,8 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { jTextArea1.setWrapStyleWord(true); jScrollPane2.setViewportView(jTextArea1); + org.openide.awt.Mnemonics.setLocalizedText(postHitCheckBox, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.postHitCheckBox.text")); // NOI18N + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -151,8 +158,8 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGroup(layout.createSequentialGroup() .addGap(18, 18, 18) .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addGroup(layout.createSequentialGroup() @@ -173,15 +180,20 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGap(5, 5, 5) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 84, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(jTextField2, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))))) - .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() - .addGap(6, 6, 6) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) - .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(layout.createSequentialGroup() + .addComponent(jTextField2, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGap(2, 2, 2) .addComponent(mimeTypeLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 181, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 181, javax.swing.GroupLayout.PREFERRED_SIZE)))) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(6, 6, 6) + .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(layout.createSequentialGroup() + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(postHitCheckBox))) .addGap(0, 0, Short.MAX_VALUE))) .addContainerGap()))) ); @@ -191,11 +203,11 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGap(16, 16, 16) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(layout.createSequentialGroup() + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(mimeTypeLabel) .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) @@ -212,7 +224,8 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(offsetLabel)) - .addGap(12, 12, 12))) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(postHitCheckBox))) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) .addGroup(layout.createSequentialGroup() .addGap(18, 18, 18) @@ -244,6 +257,7 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { private javax.swing.JTextField mimeTypeTextField; private javax.swing.JLabel offsetLabel; private javax.swing.JTextField offsetTextField; + private javax.swing.JCheckBox postHitCheckBox; private javax.swing.JLabel signatureLabel; private javax.swing.JComboBox signatureTypeComboBox; private javax.swing.JLabel signatureTypeLabel; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java index d96d7b4f5f..860a6b65cf 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -29,7 +29,7 @@ class UserDefinedFileTypeIdentifier { private static final String USER_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; private static final String AUTOPSY_DEFINITIONS_FILE = "AutopsyFileTypeDefinitions.xml"; - private final List signatures; + private final List signatures; /** * Creates an object that does file type identification with user-defined @@ -69,7 +69,7 @@ class UserDefinedFileTypeIdentifier { */ String identify(AbstractFile file) { String type = ""; - for (UserDefinedFileTypes.FileSignature signature : this.signatures) { + for (UserDefinedFileTypes.FileTypeSignature signature : this.signatures) { if (signature.containedIn(file)) { type = signature.getMimeType(); break; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java index fcd275a89f..690433b416 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java @@ -18,39 +18,62 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.logging.Level; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.XMLUtil; -import org.sleuthkit.autopsy.externalresults.ExternalResultsXMLParser; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.NodeList; import org.xml.sax.SAXException; +import javax.xml.bind.DatatypeConverter; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; /** * RJCTODO */ public class UserDefinedFileTypes { + private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; // NON-NLS private static final String ROOT_FILE_TYPES_ELEMENT = "fileTypes"; // NON-NLS private static final String FILE_TYPE_ELEMENT = "fileType"; // NON-NLS private static final String MIME_TYPE_ELEMENT = "mimeType"; // NON-NLS private static final String SIGNATURE_ELEMENT = "signature"; // NON-NLS private static final String OFFSET_ELEMENT = "offset"; + private static final String ENCODING = "UTF-8"; //NON-NLS // RJCTODO: Is this right? + + /** + * Reads the user-defined file type definitions from the user-defined file + * types file. + * + * @return A list of file type signature + */ + synchronized static List getUserDefinedFileTypeSignatures() { + Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); + String filePathString = filePath.toAbsolutePath().toString(); + File file = new File(filePathString); + if (file.exists() && file.canRead()) { + return UserDefinedFileTypes.Reader.readFileTypes(filePathString); + } + return Collections.emptyList(); + } /** * An association between a MIME type and a signature within a file. */ - static interface FileSignature { + static interface FileTypeSignature { /** * Gets the MIME type associated with this signature. @@ -59,6 +82,20 @@ public class UserDefinedFileTypes { */ String getMimeType(); + /** + * RJCTODO + * + * @return + */ + byte[] getSignatureBytes(); + + /** + * RJCTODO + * + * @return + */ + long getOffset(); + /** * Determines whether or not a file contains the signature. * @@ -73,7 +110,7 @@ public class UserDefinedFileTypes { * An association between a MIME type and a byte signature at a specified * offset within a file. */ - static class ByteSignature implements FileSignature { + static class ByteSignature implements FileTypeSignature { private final String mimeType; private final byte[] signatureBytes; @@ -103,6 +140,22 @@ public class UserDefinedFileTypes { return this.mimeType; } + /** + * @inheritDoc + */ + @Override + public byte[] getSignatureBytes() { + return this.signatureBytes; + } + + /** + * @inheritDoc + */ + @Override + public long getOffset() { + return this.offset; + } + /** * @inheritDoc */ @@ -117,7 +170,40 @@ public class UserDefinedFileTypes { } } - + + /** + * RJCTODO + */ + static class AsciiStringSignature implements FileTypeSignature { + + /** + * @inheritDoc + */ + @Override + public String getMimeType() { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public byte[] getSignatureBytes() { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public long getOffset() { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + /** + * @inheritDoc + */ + @Override + public boolean containedIn(AbstractFile file) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + } + /** * Provides a mechanism for persisting user-defined file types. */ @@ -129,10 +215,39 @@ public class UserDefinedFileTypes { * @param signatures A collection of file signatures. * @param filePath The path to the file. */ - static void writeFileTypes(List signatures, String filePath) { + static void writeFileTypes(List signatures, String filePath) { + try { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document doc = builder.newDocument(); + Element rootElem = doc.createElement(UserDefinedFileTypes.ROOT_FILE_TYPES_ELEMENT); + doc.appendChild(rootElem); + for (FileTypeSignature signature : signatures) { + UserDefinedFileTypes.Writer.writeFileType(signature, rootElem, doc); + } + if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypes.ENCODING, doc)) { + // RJCTODO + } + } catch (ParserConfigurationException ex) { + } } - + + static void writeFileType(FileTypeSignature signature, Element rootElem, Document doc) { + Element mimeTypeElem = doc.createElement(UserDefinedFileTypes.MIME_TYPE_ELEMENT); + mimeTypeElem.setTextContent(signature.getMimeType()); + Element sigElem = doc.createElement(UserDefinedFileTypes.SIGNATURE_ELEMENT); + sigElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); + Element offsetElem = doc.createElement(UserDefinedFileTypes.OFFSET_ELEMENT); + offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); + Element fileTypeElem = doc.createElement(UserDefinedFileTypes.FILE_TYPE_ELEMENT); + fileTypeElem.appendChild(mimeTypeElem); + fileTypeElem.appendChild(sigElem); + fileTypeElem.appendChild(offsetElem); + rootElem.appendChild(fileTypeElem); + // RJCTODO: Surely there are some exceptions that could be thrown here? + } + } /** @@ -146,28 +261,77 @@ public class UserDefinedFileTypes { * @param filePath The path to the file. * @return A collection of file signatures. */ - static List readFileTypes(String filePath) { - // RJCTODO: - try { - DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = builderFactory.newDocumentBuilder(); - Document doc = builder.parse(new FileInputStream(filePath)); - final Document doc = XMLUtil.loadDoc(ExternalResultsXMLParser.class, this.resultsFilePath, XSD_FILE); - if (doc != null) { - final Element rootElem = doc.getDocumentElement(); - if (rootElem != null && rootElem.getNodeName().equals(ExternalResultsXMLParser.TagNames.ROOT_ELEM.toString())) { - - } catch (ParserConfigurationException e) { - } catch (SAXException e) { - } catch (IOException e) { - } - - - - + static List readFileTypes(String filePath) { + try { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document doc = builder.parse(new FileInputStream(filePath)); + if (doc != null) { + Element rootElem = doc.getDocumentElement(); + if (rootElem != null && rootElem.getNodeName().equals(UserDefinedFileTypes.ROOT_FILE_TYPES_ELEMENT)) { + return UserDefinedFileTypes.Reader.parseFileTypes(rootElem); + } + } + } catch (ParserConfigurationException | SAXException | IOException e) { + // RJCTODO: + } return Collections.emptyList(); } + /** + * RJCTODO + * + * @param rootElem + * @return + */ + private static List parseFileTypes(Element rootElem) { + List signatures = new ArrayList<>(); + NodeList fileTypeElems = rootElem.getElementsByTagName(UserDefinedFileTypes.FILE_TYPE_ELEMENT); + for (int i = 0; i < fileTypeElems.getLength(); ++i) { + Element fileTypeElem = (Element) fileTypeElems.item(i); + UserDefinedFileTypes.Reader.tryParseSignature(fileTypeElem, signatures); + } + return signatures; + } + + /** + * RJCTODO + * + * @param fileTypeElem + * @param signatures + */ + private static void tryParseSignature(Element fileTypeElem, List signatures) { + try { + String mimeType = ""; + byte[] signature = null; + long offset = -1; + NodeList childElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypes.FILE_TYPE_ELEMENT); + for (int i = 0; i < childElems.getLength(); ++i) { + Element childElem = (Element) childElems.item(i); + switch (childElem.getTagName()) { + case MIME_TYPE_ELEMENT: + mimeType = childElem.getTextContent(); + break; + case SIGNATURE_ELEMENT: + signature = DatatypeConverter.parseHexBinary(childElem.getTextContent()); + break; + case OFFSET_ELEMENT: + offset = DatatypeConverter.parseLong(childElem.getTextContent()); + break; + default: + break; + } + } + if (!mimeType.isEmpty() && null != signature && signature.length > 0 && offset >= 0) { + signatures.add(new ByteSignature(mimeType, signature, offset)); + } + } catch (NumberFormatException ex) { + // RJCTODO: Log error + } catch (IllegalArgumentException ex) { + // RJCTODO: Log error + } + } + } } From 475430f9aa19f68e0634a3297287dc7f2e71c04e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 28 Nov 2014 15:01:44 -0500 Subject: [PATCH 05/84] Updates before accidental deletion of work --- .../filetypeid/FileTypeIdSettingsPanel.form | 8 +- .../filetypeid/FileTypeIdSettingsPanel.java | 98 +++-- .../UserDefinedFileTypeIdentifier.java | 30 +- .../filetypeid/UserDefinedFileTypes.java | 335 +----------------- 4 files changed, 84 insertions(+), 387 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form index efb8311caf..d20e8f8683 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form @@ -29,7 +29,7 @@ - + @@ -104,7 +104,7 @@ - + @@ -154,14 +154,14 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java index c0ffdbfd3b..af160cd19b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java @@ -18,21 +18,28 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import javax.swing.DefaultListModel; +import javax.swing.ListModel; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; -import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypes.FileTypeSignature; /** - * RJCTODO + * A panel to allow a user to make custom file type definitions. */ -class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { +final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { + +// private final HashMap fileTypeDefs; +// private final ListModel fileTypesListModel; /** - * Creates... RJCTODO + * Creates a panel to allow a user to make custom file type definitions. */ FileTypeIdSettingsPanel() { + this.fileTypeDefs = new HashMap<>(); initComponents(); - } /** @@ -40,35 +47,48 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { */ @Override public void saveSettings() { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + this.store(); } - - void load() { - List fileTypeSignatures; - - // RJCTODO read settings and initialize GUI - // Example: - // someCheckBox.setSelected(Preferences.userNodeForPackage(FileTypeIdPanel.class).getBoolean("someFlag", false)); - // or for org.openide.util with API spec. version >= 7.4: - // someCheckBox.setSelected(NbPreferences.forModule(FileTypeIdPanel.class).getBoolean("someFlag", false)); - // or: - // someTextField.setText(SomeSystemOption.getDefault().getSomeStringProperty()); - } - - void store() { - // RJCTODO store modified settings - // Example: - // Preferences.userNodeForPackage(FileTypeIdPanel.class).putBoolean("someFlag", someCheckBox.isSelected()); - // or for org.openide.util with API spec. version >= 7.4: - // NbPreferences.forModule(FileTypeIdPanel.class).putBoolean("someFlag", someCheckBox.isSelected()); - // or: - // SomeSystemOption.getDefault().setSomeStringProperty(someTextField.getText()); - } - - boolean valid() { - return true; - } + /** + * RJCTODO + */ + void load() { +// try { +// this.fileTypeDefs.clear(); +// DefaultListModel fileTypesListModel = new DefaultListModel<>(); +// for (FileTypeDefinition fileTypeDef : FileTypeDefinitionsManager.getFileTypeDefinitions()) { +// this.fileTypeDefs.put(fileTypeDef.getTypeName(), fileTypeDef); +// fileTypesListModel.addElement(fileTypeDef.getTypeName()); +// } +// this.typesList.setModel(fileTypesListModel); +// } catch (IOException ex) { +// // RJCTODO +// } + } + + /** + * RJCTODO + */ + void store() { +// try { +// ListModel fileTypesListModel = this.typesList.getModel(); +// List newFileTypeDefs = new ArrayList<>(); +// for (int i = 0; i < fileTypesListModel.getSize(); ++i) { +// String typeName = fileTypesListModel.getElementAt(i); +// newFileTypeDefs.add(this.fileTypeDefs.get(typeName)); +// } +// FileTypeDefinitionsManager.setFileTypeDefinitions(newFileTypeDefs); +// } catch (IOException ex) { +// // RJCTODO +// } + } + + boolean valid() { + // RJCTODO + return true; + } + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -79,8 +99,8 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { private void initComponents() { buttonGroup1 = new javax.swing.ButtonGroup(); - jScrollPane1 = new javax.swing.JScrollPane(); - jList1 = new javax.swing.JList(); + typesScrollPane = new javax.swing.JScrollPane(); + typesList = new javax.swing.JList(); jSeparator1 = new javax.swing.JSeparator(); mimeTypeLabel = new javax.swing.JLabel(); mimeTypeTextField = new javax.swing.JTextField(); @@ -98,7 +118,7 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { jTextArea1 = new javax.swing.JTextArea(); postHitCheckBox = new javax.swing.JCheckBox(); - jScrollPane1.setViewportView(jList1); + typesScrollPane.setViewportView(typesList); jSeparator1.setOrientation(javax.swing.SwingConstants.VERTICAL); @@ -149,7 +169,7 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addComponent(jButton1) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(jButton2)) - .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)) + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(37, 37, 37) @@ -204,7 +224,7 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addGap(18, 18, 18) @@ -247,8 +267,6 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { private javax.swing.JButton jButton1; private javax.swing.JButton jButton2; private javax.swing.JButton jButton3; - private javax.swing.JList jList1; - private javax.swing.JScrollPane jScrollPane1; private javax.swing.JScrollPane jScrollPane2; private javax.swing.JSeparator jSeparator1; private javax.swing.JTextArea jTextArea1; @@ -261,6 +279,8 @@ class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { private javax.swing.JLabel signatureLabel; private javax.swing.JComboBox signatureTypeComboBox; private javax.swing.JLabel signatureTypeLabel; + private javax.swing.JList typesList; + private javax.swing.JScrollPane typesScrollPane; // End of variables declaration//GEN-END:variables } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java index 860a6b65cf..5e9e7047c1 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -23,19 +23,17 @@ import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * Does file type identification with user-defined file type signatures. + * Does file type identification with user-defined file type fileTypeDefs. */ class UserDefinedFileTypeIdentifier { - private static final String USER_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; - private static final String AUTOPSY_DEFINITIONS_FILE = "AutopsyFileTypeDefinitions.xml"; - private final List signatures; + private final List fileTypeDefs; /** * Creates an object that does file type identification with user-defined - * file type signatures. + file type fileTypeDefs. * - * @param sigFilePath A path to a signature definitions file. + * @param sigFilePath A path to a fileTypeDef definitions file. */ static UserDefinedFileTypeIdentifier createDetector(String sigFilePath) { UserDefinedFileTypeIdentifier detector = new UserDefinedFileTypeIdentifier(); @@ -45,33 +43,35 @@ class UserDefinedFileTypeIdentifier { /** * Create an object that does file type identification with user-defined - * file type signatures. + file type fileTypeDefs. */ private UserDefinedFileTypeIdentifier() { - this.signatures = new ArrayList<>(); + this.fileTypeDefs = new ArrayList<>(); } /** - * Loads a set of user-defined file type signatures from a file. + * Loads a set of user-defined file type fileTypeDefs from a file. * - * @param sigFilePath The path to the signature definitions file. + * @param sigFilePath The path to the fileTypeDef definitions file. */ private void loadSignatures(String sigFilePath) { - // RJCTODO: Load signature file, creating + // RJCTODO: Load fileTypeDef file, creating } /** * Attempts to identify a file using the set of user-defined file type - * signatures. + fileTypeDefs. * * @param file The file to type. * @return A MIME type string or the empty string if identification fails. */ String identify(AbstractFile file) { String type = ""; - for (UserDefinedFileTypes.FileTypeSignature signature : this.signatures) { - if (signature.containedIn(file)) { - type = signature.getMimeType(); + for (FileTypeDefinitionsManager.FileTypeDefinition fileTypeDef : this.fileTypeDefs) { + if (fileTypeDef.matches(file)) { + type = fileTypeDef.getTypeName(); + // RJCTODO: Add attribute to GEN IBNFO artifact? + // RJCTODO: Handle alert here? break; } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java index 690433b416..4c2ca9044f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java @@ -1,337 +1,14 @@ /* - * Autopsy Forensic Browser - * - * Copyright 2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. */ package org.sleuthkit.autopsy.modules.filetypeid; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; -import javax.xml.bind.DatatypeConverter; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.coreutils.XMLUtil; -import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; - /** - * RJCTODO + * + * @author rcordovano */ public class UserDefinedFileTypes { - - private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; // NON-NLS - private static final String ROOT_FILE_TYPES_ELEMENT = "fileTypes"; // NON-NLS - private static final String FILE_TYPE_ELEMENT = "fileType"; // NON-NLS - private static final String MIME_TYPE_ELEMENT = "mimeType"; // NON-NLS - private static final String SIGNATURE_ELEMENT = "signature"; // NON-NLS - private static final String OFFSET_ELEMENT = "offset"; - private static final String ENCODING = "UTF-8"; //NON-NLS // RJCTODO: Is this right? - - /** - * Reads the user-defined file type definitions from the user-defined file - * types file. - * - * @return A list of file type signature - */ - synchronized static List getUserDefinedFileTypeSignatures() { - Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); - String filePathString = filePath.toAbsolutePath().toString(); - File file = new File(filePathString); - if (file.exists() && file.canRead()) { - return UserDefinedFileTypes.Reader.readFileTypes(filePathString); - } - return Collections.emptyList(); - } - - /** - * An association between a MIME type and a signature within a file. - */ - static interface FileTypeSignature { - - /** - * Gets the MIME type associated with this signature. - * - * @return The MIME type string. - */ - String getMimeType(); - - /** - * RJCTODO - * - * @return - */ - byte[] getSignatureBytes(); - - /** - * RJCTODO - * - * @return - */ - long getOffset(); - - /** - * Determines whether or not a file contains the signature. - * - * @param file The file to test. - * @return True or false. - */ - boolean containedIn(AbstractFile file); - - } - - /** - * An association between a MIME type and a byte signature at a specified - * offset within a file. - */ - static class ByteSignature implements FileTypeSignature { - - private final String mimeType; - private final byte[] signatureBytes; - private final long offset; - private final byte[] buffer; - - /** - * Creates an association between a MIME type and a byte signature at a - * specified offset. - * - * @param mimeType The MIME type string. - * @param signatureBytes The bytes of the signature. - * @param offset The offset of the signature within a file. - */ - private ByteSignature(String mimeType, byte[] signatureBytes, long offset) { - this.mimeType = mimeType; - this.signatureBytes = signatureBytes; - this.offset = offset; - this.buffer = new byte[signatureBytes.length]; - } - - /** - * @inheritDoc - */ - @Override - public String getMimeType() { - return this.mimeType; - } - - /** - * @inheritDoc - */ - @Override - public byte[] getSignatureBytes() { - return this.signatureBytes; - } - - /** - * @inheritDoc - */ - @Override - public long getOffset() { - return this.offset; - } - - /** - * @inheritDoc - */ - @Override - public boolean containedIn(AbstractFile file) { - try { - int bytesRead = file.read(this.buffer, offset, buffer.length); - return bytesRead == buffer.length ? Arrays.equals(this.buffer, this.signatureBytes) : false; - } catch (TskCoreException ex) { - return false; - } - } - - } - - /** - * RJCTODO - */ - static class AsciiStringSignature implements FileTypeSignature { - - /** - * @inheritDoc - */ - @Override - public String getMimeType() { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. - } - - @Override - public byte[] getSignatureBytes() { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. - } - - @Override - public long getOffset() { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. - } - - /** - * @inheritDoc - */ - @Override - public boolean containedIn(AbstractFile file) { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. - } - - } - - /** - * Provides a mechanism for persisting user-defined file types. - */ - static class Writer { - - /** - * Reads user-defined file type definitions from a file. - * - * @param signatures A collection of file signatures. - * @param filePath The path to the file. - */ - static void writeFileTypes(List signatures, String filePath) { - try { - DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = builderFactory.newDocumentBuilder(); - Document doc = builder.newDocument(); - Element rootElem = doc.createElement(UserDefinedFileTypes.ROOT_FILE_TYPES_ELEMENT); - doc.appendChild(rootElem); - for (FileTypeSignature signature : signatures) { - UserDefinedFileTypes.Writer.writeFileType(signature, rootElem, doc); - } - if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypes.ENCODING, doc)) { - // RJCTODO - } - } catch (ParserConfigurationException ex) { - } - - } - - static void writeFileType(FileTypeSignature signature, Element rootElem, Document doc) { - Element mimeTypeElem = doc.createElement(UserDefinedFileTypes.MIME_TYPE_ELEMENT); - mimeTypeElem.setTextContent(signature.getMimeType()); - Element sigElem = doc.createElement(UserDefinedFileTypes.SIGNATURE_ELEMENT); - sigElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); - Element offsetElem = doc.createElement(UserDefinedFileTypes.OFFSET_ELEMENT); - offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); - Element fileTypeElem = doc.createElement(UserDefinedFileTypes.FILE_TYPE_ELEMENT); - fileTypeElem.appendChild(mimeTypeElem); - fileTypeElem.appendChild(sigElem); - fileTypeElem.appendChild(offsetElem); - rootElem.appendChild(fileTypeElem); - // RJCTODO: Surely there are some exceptions that could be thrown here? - } - - } - - /** - * Provides a method for reading persisted user-defined file types. - */ - static class Reader { - - /** - * Reads user-defined file type definitions from a file. - * - * @param filePath The path to the file. - * @return A collection of file signatures. - */ - static List readFileTypes(String filePath) { - try { - DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = builderFactory.newDocumentBuilder(); - Document doc = builder.parse(new FileInputStream(filePath)); - if (doc != null) { - Element rootElem = doc.getDocumentElement(); - if (rootElem != null && rootElem.getNodeName().equals(UserDefinedFileTypes.ROOT_FILE_TYPES_ELEMENT)) { - return UserDefinedFileTypes.Reader.parseFileTypes(rootElem); - } - } - } catch (ParserConfigurationException | SAXException | IOException e) { - // RJCTODO: - } - return Collections.emptyList(); - } - - /** - * RJCTODO - * - * @param rootElem - * @return - */ - private static List parseFileTypes(Element rootElem) { - List signatures = new ArrayList<>(); - NodeList fileTypeElems = rootElem.getElementsByTagName(UserDefinedFileTypes.FILE_TYPE_ELEMENT); - for (int i = 0; i < fileTypeElems.getLength(); ++i) { - Element fileTypeElem = (Element) fileTypeElems.item(i); - UserDefinedFileTypes.Reader.tryParseSignature(fileTypeElem, signatures); - } - return signatures; - } - - /** - * RJCTODO - * - * @param fileTypeElem - * @param signatures - */ - private static void tryParseSignature(Element fileTypeElem, List signatures) { - try { - String mimeType = ""; - byte[] signature = null; - long offset = -1; - NodeList childElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypes.FILE_TYPE_ELEMENT); - for (int i = 0; i < childElems.getLength(); ++i) { - Element childElem = (Element) childElems.item(i); - switch (childElem.getTagName()) { - case MIME_TYPE_ELEMENT: - mimeType = childElem.getTextContent(); - break; - case SIGNATURE_ELEMENT: - signature = DatatypeConverter.parseHexBinary(childElem.getTextContent()); - break; - case OFFSET_ELEMENT: - offset = DatatypeConverter.parseLong(childElem.getTextContent()); - break; - default: - break; - } - } - if (!mimeType.isEmpty() && null != signature && signature.length > 0 && offset >= 0) { - signatures.add(new ByteSignature(mimeType, signature, offset)); - } - } catch (NumberFormatException ex) { - // RJCTODO: Log error - } catch (IllegalArgumentException ex) { - // RJCTODO: Log error - } - } - - } - + } From a333d97c05e1cfa815b8530cae945f49692872c5 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sat, 29 Nov 2014 14:09:28 -0500 Subject: [PATCH 06/84] Continue work on user defined file types feature --- .../sleuthkit/autopsy/coreutils/XMLUtil.java | 12 + .../autopsy/modules/filetypeid/FileType.java | 152 ++++++++ .../filetypeid/FileTypeIdSettingsPanel.java | 60 ++-- .../UserDefinedFileTypeIdentifier.java | 49 +-- .../filetypeid/UserDefinedFileTypes.java | 338 +++++++++++++++++- .../filetypeid/UserDefinedFileTypes.xsd | 50 +++ 6 files changed, 594 insertions(+), 67 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java index a997f01dd5..dc12e3bd2c 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java @@ -54,6 +54,18 @@ import org.xml.sax.SAXException; */ public class XMLUtil { + /** + * Creates a W3C DOM document. + * + * @return The document object. + * @throws ParserConfigurationException + */ + public static Document createDoc() throws ParserConfigurationException { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + return builder.newDocument(); + } + /** * Utility to validate XML files against pre-defined schema files. * diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java new file mode 100644 index 0000000000..d4fc484673 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -0,0 +1,152 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.util.Arrays; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Represents a named file type characterized by a file signature. + */ +class FileType { + + private final String typeName; + private final Signature signature; + private final boolean alert; + + /** + * Creates a representation of a named file type characterized by a file + * signature. + * + * @param typeName The name of the file type. + * @param signature The signature that characterizes the file type. + * @param alert A flag indicating whether the user wishes to be alerted when + * a file matching this type is encountered. + */ + FileType(String typeName, Signature signature, boolean alert) { + this.typeName = typeName; + this.signature = signature; + this.alert = alert; + } + + /** + * Gets the name associated with this file type. + * + * @return The type name. + */ + String getTypeName() { + return this.typeName; + } + + /** + * Gets the signature associated with this file type. + * + * @return The file signature. + */ + Signature getSignature() { + return this.signature; + } + + /** + * Determines whether or not a given file is an instance of this file type. + * + * @param file The file to test + * @return True or false. + */ + boolean matches(AbstractFile file) { + return this.signature.containedIn(file); + } + + /** + * Indicates whether or not an alert is desired if a file of this type is + * encountered. + * + * @return True or false. + */ + boolean alertOnMatch() { + return this.alert; + } + + /** + * Represents a file signature consisting of a sequence of bytes at a + * specific offset within a file. + */ + static class Signature { + + /** + * The way the signature byte sequence should be interpreted. + */ + enum Type { + + RAW, ASCII + }; + + private final byte[] signatureBytes; + private final long offset; + private final Type type; + + /** + * Creates a representation of a file signature consisting of a sequence + * of bytes at a specific offset within a file. + * + * @param signatureBytes The signature bytes + * @param offset The offset of the signature bytes. + * @param type The interpretation of the signature bytes (e.g., raw + * bytes, an ASCII string). + */ + Signature(byte[] signatureBytes, long offset, Type type) { + this.signatureBytes = signatureBytes; + this.offset = offset; + this.type = type; + } + + /** + * Gets the byte sequence of the signature. + * + * @return The byte sequence as an array of bytes. + */ + byte[] getSignatureBytes() { + return Arrays.copyOf(this.signatureBytes, this.signatureBytes.length); + } + + /** + * Gets the offset of the signature. + * + * @return The offset. + */ + long getOffset() { + return this.offset; + } + + /** + * Gets the interpretation of the byte sequence for the signature. + * + * @return The signature type. + */ + Type getType() { + return this.type; + } + + /** + * Determines whether or not the signature is contained within a given + * file. + * + * @param file The file to test + * @return True or false. + */ + boolean containedIn(AbstractFile file) { + try { + byte[] buffer = new byte[this.signatureBytes.length]; + int bytesRead = file.read(buffer, offset, this.signatureBytes.length); + return ((bytesRead == this.signatureBytes.length) && (Arrays.equals(buffer, this.signatureBytes))); + } catch (TskCoreException ex) { + return false; + } + } + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java index af160cd19b..ec6598b838 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java @@ -18,27 +18,28 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import javax.swing.DefaultListModel; import javax.swing.ListModel; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; +//import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypes.FileType; +import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypes.UserDefinedFileTypesException; /** * A panel to allow a user to make custom file type definitions. */ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { - -// private final HashMap fileTypeDefs; + + private final HashMap fileTypes; // private final ListModel fileTypesListModel; /** * Creates a panel to allow a user to make custom file type definitions. */ FileTypeIdSettingsPanel() { - this.fileTypeDefs = new HashMap<>(); + this.fileTypes = new HashMap<>(); initComponents(); } @@ -49,41 +50,44 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { public void saveSettings() { this.store(); } - + /** * RJCTODO */ void load() { -// try { -// this.fileTypeDefs.clear(); -// DefaultListModel fileTypesListModel = new DefaultListModel<>(); -// for (FileTypeDefinition fileTypeDef : FileTypeDefinitionsManager.getFileTypeDefinitions()) { -// this.fileTypeDefs.put(fileTypeDef.getTypeName(), fileTypeDef); -// fileTypesListModel.addElement(fileTypeDef.getTypeName()); -// } -// this.typesList.setModel(fileTypesListModel); -// } catch (IOException ex) { -// // RJCTODO -// } + try { + this.fileTypes.clear(); + DefaultListModel fileTypesListModel = new DefaultListModel<>(); + for (FileType fileType : UserDefinedFileTypes.getFileTypes()) { + this.fileTypes.put(fileType.getTypeName(), fileType); + fileTypesListModel.addElement(fileType.getTypeName()); + } + this.typesList.setModel(fileTypesListModel); + } catch (UserDefinedFileTypesException ex) { + // RJCTODO + } } /** * RJCTODO */ void store() { -// try { -// ListModel fileTypesListModel = this.typesList.getModel(); -// List newFileTypeDefs = new ArrayList<>(); -// for (int i = 0; i < fileTypesListModel.getSize(); ++i) { -// String typeName = fileTypesListModel.getElementAt(i); -// newFileTypeDefs.add(this.fileTypeDefs.get(typeName)); -// } -// FileTypeDefinitionsManager.setFileTypeDefinitions(newFileTypeDefs); -// } catch (IOException ex) { -// // RJCTODO -// } + try { + ListModel fileTypesListModel = this.typesList.getModel(); + List newFileTypes = new ArrayList<>(); + for (int i = 0; i < fileTypesListModel.getSize(); ++i) { + String typeName = fileTypesListModel.getElementAt(i); + newFileTypes.add(this.fileTypes.get(typeName)); + } + UserDefinedFileTypes.setFileTypes(newFileTypes); + } catch (UserDefinedFileTypesException ex) { + // RJCTODO + } } - + + /** + * RJCTODO + */ boolean valid() { // RJCTODO return true; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java index 5e9e7047c1..5a5ad78f31 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -23,55 +23,40 @@ import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * Does file type identification with user-defined file type fileTypeDefs. + * Does file type identification with user-defined file types. */ -class UserDefinedFileTypeIdentifier { +final class UserDefinedFileTypeIdentifier { - private final List fileTypeDefs; + private final List fileTypes; /** * Creates an object that does file type identification with user-defined - file type fileTypeDefs. - * - * @param sigFilePath A path to a fileTypeDef definitions file. + * file types. Does not load the file type definitions. */ - static UserDefinedFileTypeIdentifier createDetector(String sigFilePath) { - UserDefinedFileTypeIdentifier detector = new UserDefinedFileTypeIdentifier(); - detector.loadSignatures(sigFilePath); - return detector; + UserDefinedFileTypeIdentifier() { + this.fileTypes = new ArrayList<>(); } /** - * Create an object that does file type identification with user-defined - file type fileTypeDefs. + * Loads the set of user-defined file types. */ - private UserDefinedFileTypeIdentifier() { - this.fileTypeDefs = new ArrayList<>(); - } - - /** - * Loads a set of user-defined file type fileTypeDefs from a file. - * - * @param sigFilePath The path to the fileTypeDef definitions file. - */ - private void loadSignatures(String sigFilePath) { - // RJCTODO: Load fileTypeDef file, creating + void loadFileTypes() throws UserDefinedFileTypes.UserDefinedFileTypesException { + this.fileTypes.clear(); + this.fileTypes.addAll(UserDefinedFileTypes.getFileTypes()); } /** * Attempts to identify a file using the set of user-defined file type - fileTypeDefs. + * file types. * * @param file The file to type. - * @return A MIME type string or the empty string if identification fails. + * @return A FileType object or null if identification fails. */ - String identify(AbstractFile file) { - String type = ""; - for (FileTypeDefinitionsManager.FileTypeDefinition fileTypeDef : this.fileTypeDefs) { - if (fileTypeDef.matches(file)) { - type = fileTypeDef.getTypeName(); - // RJCTODO: Add attribute to GEN IBNFO artifact? - // RJCTODO: Handle alert here? + FileType identify(AbstractFile file) { + FileType type = null; + for (FileType fileType : this.fileTypes) { + if (fileType.matches(file)) { + type = fileType; break; } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java index 4c2ca9044f..84bc4f8778 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java @@ -1,14 +1,338 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import javax.xml.bind.DatatypeConverter; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; +import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; + /** - * - * @author rcordovano + * Allows a user to define named file types characterized by file signatures. */ -public class UserDefinedFileTypes { - +final class UserDefinedFileTypes { + + private static final Logger logger = Logger.getLogger(UserDefinedFileTypes.class.getName()); + private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; // NON-NLS + private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; // NON-NLS + private static final String AUTOPSY_TYPE_DEFINITIONS_FILE = "AutopsyFileTypeDefinitions.xml"; // NON-NLS + private static final String FILE_TYPES_TAG_NAME = "filetypes"; // NON-NLS + private static final String FILE_TYPE_TAG_NAME = "filetype"; // NON-NLS + private static final String ALERT_ATTRIBUTE = "alert"; // NON-NLS + private static final String TYPE_NAME_TAG_NAME = "typename"; // NON-NLS + private static final String SIGNATURE_TAG_NAME = "signature"; // NON-NLS + private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; // NON-NLS + private static final String BYTES_TAG_NAME = "bytes"; // NON-NLS + private static final String OFFSET_TAG_NAME = "offset"; // NON-NLS + private static final String ENCODING = "UTF-8"; //NON-NLS // RJCTODO: Is this right? + + static { + UserDefinedFileTypes.writePredefinedTypes(); + } + + /** + * Writes the predefined file types definition file. + */ + private static void writePredefinedTypes() { + try { + Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); + String filePathString = filePath.toAbsolutePath().toString(); + List fileTypes = new ArrayList<>(); // RJCTODO + UserDefinedFileTypes.writeFileTypes(fileTypes, filePathString); + } catch (UserDefinedFileTypesException ex) { + // RJCTODO + } + } + + /** + * Gets the user-defined file types. + * + * @return A list of file types, possibly empty + */ + synchronized static List getFileTypes() throws UserDefinedFileTypesException { + Map fileTypes = new HashMap<>(); + UserDefinedFileTypes.readFileTypes(fileTypes, UserDefinedFileTypes.AUTOPSY_TYPE_DEFINITIONS_FILE); + UserDefinedFileTypes.readFileTypes(fileTypes, UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); + return new ArrayList(fileTypes.values()); + } + + /** + * Reads file type definitions from a file into a map of type names to file + * types. File types already loaded into the map will be overwritten by file + * types from the file if the type name is the same. + * + * @param fileTypes The map of type names to file type objects. + * @param fileName The file from which to read the file type definitions. + */ + private static void readFileTypes(Map fileTypes, String fileName) throws UserDefinedFileTypesException { + Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), fileName); + String filePathString = filePath.toAbsolutePath().toString(); + File file = new File(filePathString); + if (file.exists() && file.canRead()) { + for (FileType fileType : UserDefinedFileTypes.XMLReader.readFileTypes(filePathString)) { + fileTypes.put(fileType.getTypeName(), fileType); + } + } + } + + /** + * Sets the user-defined file types. + * + * @param fileTypes The file type definitions. + * @throws UserDefinedFileTypesException + */ + synchronized static void setFileTypes(List fileTypes) throws UserDefinedFileTypesException { + Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); + String filePathString = filePath.toAbsolutePath().toString(); + UserDefinedFileTypes.writeFileTypes(fileTypes, filePathString); + } + + /** + * Writes a set of file type definitions to a given file. + * + * @param fileTypes The file types. + * @param filePaht The destination file. + * @throws UserDefinedFileTypesException + */ + private static void writeFileTypes(List fileTypes, String filePath) throws UserDefinedFileTypesException { + try { + UserDefinedFileTypes.XMLWriter.writeFileTypes(fileTypes, filePath); + } catch (ParserConfigurationException | IOException ex) { + UserDefinedFileTypes.logger.log(Level.SEVERE, "Failed to write file types file", ex); + throw new UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message + } + } + + /** + * Provides a mechanism for writing a set of file type definitions to an XML + * file. + */ + private static class XMLWriter { + + /** + * Writes a set of file type definitions to an XML file. + * + * @param signatures A collection of file types. + * @param filePath The path to the destination file. + */ + private static void writeFileTypes(List fileTypes, String filePath) throws ParserConfigurationException, IOException { + Document doc = XMLUtil.createDoc(); + Element fileTypesElem = doc.createElement(UserDefinedFileTypes.FILE_TYPES_TAG_NAME); + doc.appendChild(fileTypesElem); + for (FileType fileType : fileTypes) { + Element fileTypeElem = UserDefinedFileTypes.XMLWriter.createFileTypeElement(fileType, doc); + fileTypesElem.appendChild(fileTypeElem); + } + if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypes.ENCODING, doc)) { + throw new IOException("Could not save user defined file types"); + } + } + + /** + * Creates an XML representation of a file type. + * + * @param fileType The file type object. + * @param doc The DOM document to use to create the XML. + * @return An XML element. + */ + private static Element createFileTypeElement(FileType fileType, Document doc) { + /** + * Create a file type element with an alert attribute. + */ + Element fileTypeElem = doc.createElement(UserDefinedFileTypes.FILE_TYPE_TAG_NAME); + fileTypeElem.setAttribute(UserDefinedFileTypes.ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); + + /** + * Add a type name child element. + */ + Element typeNameElem = doc.createElement(UserDefinedFileTypes.TYPE_NAME_TAG_NAME); + typeNameElem.setTextContent(fileType.getTypeName()); + fileTypeElem.appendChild(typeNameElem); + + /** + * Add a signature child element with a type attribute. + */ + Signature signature = fileType.getSignature(); + Element signatureElem = doc.createElement(UserDefinedFileTypes.SIGNATURE_TAG_NAME); + signatureElem.setAttribute(UserDefinedFileTypes.SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); + fileTypeElem.appendChild(signatureElem); + + /** + * Add a bytes child element to the signature element. + */ + Element bytesElem = doc.createElement(UserDefinedFileTypes.BYTES_TAG_NAME); + bytesElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); + signatureElem.appendChild(bytesElem); + + /** + * Add an offset child element to the signature element. + */ + Element offsetElem = doc.createElement(UserDefinedFileTypes.OFFSET_TAG_NAME); + offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); + signatureElem.appendChild(offsetElem); + + return fileTypeElem; + } + } + + /** + * Provides a mechanism for reading a set of file type definitions from an + * XML file. + */ + private static class XMLReader { + + /** + * Reads a set of file type definitions from an XML file. + * + * @param filePath The path to the file. + * @return A collection of file types. + */ + private static List readFileTypes(String filePath) throws UserDefinedFileTypesException { + List fileTypes = new ArrayList<>(); + Path xsdPath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); + String xsdPathString = xsdPath.toAbsolutePath().toString(); + File file = new File(xsdPathString); + if (file.exists() && file.canRead()) { + Document doc = XMLUtil.loadDoc(UserDefinedFileTypes.XMLReader.class, filePath, xsdPathString); + if (doc != null) { + Element fileTypesElem = doc.getDocumentElement(); + if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypes.FILE_TYPES_TAG_NAME)) { + NodeList fileTypeElems = fileTypesElem.getElementsByTagName(UserDefinedFileTypes.FILE_TYPE_TAG_NAME); + for (int i = 0; i < fileTypeElems.getLength(); ++i) { + Element fileTypeElem = (Element) fileTypeElems.item(i); + FileType fileType = UserDefinedFileTypes.XMLReader.parseFileType(fileTypeElem); + fileTypes.add(fileType); + } + } + } + } + return fileTypes; + } + + /** + * Parse a file type definition from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return A file type object. + * @throws UserDefinedFileTypesException + */ + private static FileType parseFileType(Element fileTypeElem) throws UserDefinedFileTypesException { + /** + * Get the alert attribute. + */ + String alertAttribute = fileTypeElem.getAttribute(UserDefinedFileTypes.ALERT_ATTRIBUTE); + boolean alert = Boolean.parseBoolean(alertAttribute); + + /** + * Get the type name child element. + */ + String typeName = UserDefinedFileTypes.getChildElementTextContent(fileTypeElem, UserDefinedFileTypes.TYPE_NAME_TAG_NAME); + + /** + * Get the signature child element. + */ + Element signatureElem; + NodeList signatureElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypes.SIGNATURE_TAG_NAME); + signatureElem = (Element) signatureElems.item(0); + + /** + * Get the signature type attribute from the signature child + * element. + */ + String sigTypeAttribute = signatureElem.getAttribute(typeName); + Signature.Type signatureType = Signature.Type.valueOf(sigTypeAttribute); + + /** + * Get the signature bytes. + */ + String sigBytesString = UserDefinedFileTypes.getChildElementTextContent(signatureElem, UserDefinedFileTypes.TYPE_NAME_TAG_NAME); + byte[] signatureBytes = DatatypeConverter.parseHexBinary(sigBytesString); + + /** + * Get the offset. + */ + String offsetString = UserDefinedFileTypes.getChildElementTextContent(signatureElem, UserDefinedFileTypes.OFFSET_TAG_NAME); + long offset = DatatypeConverter.parseLong(offsetString); + + /** + * Put it all together. + */ + Signature signature = new Signature(signatureBytes, offset, signatureType); + return new FileType(typeName, signature, alert); + } + } + + /** + * Gets the text content of a single child element. + * + * @param elem The parent element. + * @param tagName The tag name of the child element. + * @return The text content. + * @throws UserDefinedFileTypesException + */ + private static String getChildElementTextContent(Element elem, String tagName) throws UserDefinedFileTypesException { + /** + * The checks here are essentially "sanity checks" since the XML was + * already validated using the XSD file. + */ + String textContent = ""; + NodeList childElems = elem.getElementsByTagName(tagName); + if (childElems.getLength() > 0) { + Element childElem = (Element) childElems.item(0); + textContent = childElem.getTextContent(); + if (textContent.isEmpty()) { + UserDefinedFileTypes.logger.log(Level.SEVERE, "File type {0} child element missing text content", tagName); // NON-NLS + } + } else { + UserDefinedFileTypes.logger.log(Level.SEVERE, "File type element missing {0} child element", tagName); // NON-NLS + } + if (textContent.isEmpty()) { + throw new UserDefinedFileTypes.UserDefinedFileTypesException("Error reading file type definitions file, see log for details"); // RJCTODO: Bundle + } + return textContent; + } + + /** + * An exception type to conflate more implementation-details-specific + * exception types (which are logged by this class) into a single generic + * type. + */ + static class UserDefinedFileTypesException extends Exception { + + UserDefinedFileTypesException(String message) { + super(message); + } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd new file mode 100644 index 0000000000..2512542294 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From f94ec08dc9a4611997c0d1a7834c3c5a78043c4e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sun, 30 Nov 2014 12:27:29 -0500 Subject: [PATCH 07/84] Changing file types management code from static to singleton --- .../modules/filetypeid/Bundle.properties | 8 +- .../autopsy/modules/filetypeid/FileType.java | 52 +- .../FileTypeIdOptionsPanelController.java | 5 +- .../filetypeid/FileTypeIdSettingsPanel.form | 32 +- .../filetypeid/FileTypeIdSettingsPanel.java | 125 +++-- .../UserDefinedFileTypeIdentifier.java | 23 +- .../filetypeid/UserDefinedFileTypes.java | 338 ------------ .../filetypeid/UserDefinedFileTypes.xsd | 51 +- .../UserDefinedFileTypesManager.java | 511 ++++++++++++++++++ 9 files changed, 671 insertions(+), 474 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 58978a9b5d..8df981383f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -8,10 +8,6 @@ FileTypeIdIngestModule.complete.srvMsg.text=File Type Id Results FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings FileTypeIdModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) -FileTypeIdSettingsPanel.jTextField2.text= -FileTypeIdSettingsPanel.jButton1.text=New Type -FileTypeIdSettingsPanel.jButton2.text=DeleteType -FileTypeIdSettingsPanel.jButton3.text=Save Type FileTypeIdSettingsPanel.mimeTypeTextField.text= FileTypeIdSettingsPanel.mimeTypeLabel.text=Mime Type FileTypeIdSettingsPanel.signatureTypeLabel.text=Signature Type @@ -21,3 +17,7 @@ FileTypeIdSettingsPanel.offsetTextField.text= FileTypeIdSettingsPanel.hexPrefixLabel.text=0x FileTypeIdSettingsPanel.jTextArea1.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. FileTypeIdSettingsPanel.postHitCheckBox.text=Post interesting file hit when found +FileTypeIdSettingsPanel.newTypeButton.text=New Type +FileTypeIdSettingsPanel.deleteTypeButton.text=DeleteType +FileTypeIdSettingsPanel.saveTypeButton.text=Save Type +FileTypeIdSettingsPanel.signatureTextField.text= diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java index d4fc484673..98b174b46f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -1,18 +1,33 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.modules.filetypeid; import java.util.Arrays; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** * Represents a named file type characterized by a file signature. */ -class FileType { +final class FileType { private final String typeName; private final Signature signature; @@ -27,9 +42,9 @@ class FileType { * @param alert A flag indicating whether the user wishes to be alerted when * a file matching this type is encountered. */ - FileType(String typeName, Signature signature, boolean alert) { + FileType(String typeName, final Signature signature, boolean alert) { this.typeName = typeName; - this.signature = signature; + this.signature = new Signature(signature.getSignatureBytes(), signature.getOffset(), signature.getType()); this.alert = alert; } @@ -48,16 +63,16 @@ class FileType { * @return The file signature. */ Signature getSignature() { - return this.signature; + return new Signature(this.signature.getSignatureBytes(), this.signature.getOffset(), this.signature.getType()); } /** * Determines whether or not a given file is an instance of this file type. * - * @param file The file to test + * @param file The file to test. * @return True or false. */ - boolean matches(AbstractFile file) { + boolean matches(final AbstractFile file) { return this.signature.containedIn(file); } @@ -72,10 +87,12 @@ class FileType { } /** - * Represents a file signature consisting of a sequence of bytes at a - * specific offset within a file. + * A file signature consisting of a sequence of bytes at a specific offset + * within a file. */ - static class Signature { + static final class Signature { + + private static final Logger logger = Logger.getLogger(Signature.class.getName()); /** * The way the signature byte sequence should be interpreted. @@ -90,16 +107,16 @@ class FileType { private final Type type; /** - * Creates a representation of a file signature consisting of a sequence - * of bytes at a specific offset within a file. + * Creates a file signature consisting of a sequence of bytes at a + * specific offset within a file. * * @param signatureBytes The signature bytes * @param offset The offset of the signature bytes. * @param type The interpretation of the signature bytes (e.g., raw * bytes, an ASCII string). */ - Signature(byte[] signatureBytes, long offset, Type type) { - this.signatureBytes = signatureBytes; + Signature(final byte[] signatureBytes, long offset, Type type) { + this.signatureBytes = Arrays.copyOf(signatureBytes, signatureBytes.length); this.offset = offset; this.type = type; } @@ -138,12 +155,13 @@ class FileType { * @param file The file to test * @return True or false. */ - boolean containedIn(AbstractFile file) { + boolean containedIn(final AbstractFile file) { try { byte[] buffer = new byte[this.signatureBytes.length]; int bytesRead = file.read(buffer, offset, this.signatureBytes.length); return ((bytesRead == this.signatureBytes.length) && (Arrays.equals(buffer, this.signatureBytes))); } catch (TskCoreException ex) { + Signature.logger.log(Level.WARNING, "Error reading from file with objId = " + file.getId(), ex); return false; } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java index 4209dd9dc5..44c7730c1a 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java @@ -45,12 +45,11 @@ public final class FileTypeIdOptionsPanelController extends OptionsPanelControll @Override public void cancel() { - // need not do anything special, if no changes have been persisted yet } @Override public boolean isValid() { - return getPanel().valid(); + return true; } @Override @@ -60,7 +59,7 @@ public final class FileTypeIdOptionsPanelController extends OptionsPanelControll @Override public HelpCtx getHelpCtx() { - return null; // new HelpCtx("...ID") if you have a help set + return null; } @Override diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form index d20e8f8683..d8ad98b738 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form @@ -25,16 +25,16 @@ - + - + - + @@ -66,7 +66,7 @@ - + @@ -119,7 +119,7 @@ - + @@ -136,13 +136,13 @@ - - + + - + @@ -196,10 +196,10 @@ - + - + @@ -217,24 +217,24 @@ - + - + - + - + - + - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java index ec6598b838..595a32397e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java @@ -18,28 +18,28 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import javax.swing.DefaultListModel; -import javax.swing.ListModel; +import javax.swing.event.ListSelectionEvent; +import javax.swing.event.ListSelectionListener; +import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; -//import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypes.FileType; -import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypes.UserDefinedFileTypesException; +import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; +import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException; /** * A panel to allow a user to make custom file type definitions. */ -final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { +final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { - private final HashMap fileTypes; -// private final ListModel fileTypesListModel; + private final HashMap fileTypes = new HashMap<>(); + private final HashMap changedFileTypes = new HashMap<>(); + private final HashMap deletedFileTypes = new HashMap<>(); /** * Creates a panel to allow a user to make custom file type definitions. */ FileTypeIdSettingsPanel() { - this.fileTypes = new HashMap<>(); initComponents(); } @@ -52,47 +52,60 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { } /** - * RJCTODO + * Populates the child components. */ - void load() { - try { - this.fileTypes.clear(); - DefaultListModel fileTypesListModel = new DefaultListModel<>(); - for (FileType fileType : UserDefinedFileTypes.getFileTypes()) { - this.fileTypes.put(fileType.getTypeName(), fileType); - fileTypesListModel.addElement(fileType.getTypeName()); - } - this.typesList.setModel(fileTypesListModel); - } catch (UserDefinedFileTypesException ex) { - // RJCTODO + @Override + public void load() { + fileTypes.clear(); + DefaultListModel fileTypesListModel = new DefaultListModel<>(); + for (FileType fileType : UserDefinedFileTypesManager.getInstance().getFileTypes()) { + fileTypes.put(fileType.getTypeName(), fileType); + fileTypesListModel.addElement(fileType.getTypeName()); + } + typesList.setModel(fileTypesListModel); + + typesList.addListSelectionListener(new TypesListSelectionListener()); + + if (!fileTypesListModel.isEmpty()) { + typesList.setSelectedIndex(0); } } /** - * RJCTODO + * Stores any changes to the user-defined types. */ - void store() { + @Override + public void store() { try { - ListModel fileTypesListModel = this.typesList.getModel(); - List newFileTypes = new ArrayList<>(); - for (int i = 0; i < fileTypesListModel.getSize(); ++i) { - String typeName = fileTypesListModel.getElementAt(i); - newFileTypes.add(this.fileTypes.get(typeName)); - } - UserDefinedFileTypes.setFileTypes(newFileTypes); + UserDefinedFileTypesManager fileTypesManager = UserDefinedFileTypesManager.getInstance(); + fileTypesManager.addFileTypes(changedFileTypes.values()); + fileTypesManager.deleteFileTypes(deletedFileTypes.values()); } catch (UserDefinedFileTypesException ex) { // RJCTODO } } - /** - * RJCTODO - */ - boolean valid() { - // RJCTODO - return true; + private class TypesListSelectionListener implements ListSelectionListener { + + @Override + public void valueChanged(ListSelectionEvent e) { + if (e.getValueIsAdjusting() == false) { + if (typesList.getSelectedIndex() == -1) { + deleteTypeButton.setEnabled(false); + + } else { + String typeName = (String) typesList.getSelectedValue(); + FileType fileType = fileTypes.get(typeName); + Signature signature = fileType.getSignature(); + mimeTypeTextField.setText(typeName); + offsetTextField.setText(Long.toString(signature.getOffset())); + deleteTypeButton.setEnabled(true); + } + } + } } + // RJCTODO: Consider fixing OptinsPanel interface or writing story /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -109,12 +122,12 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { mimeTypeLabel = new javax.swing.JLabel(); mimeTypeTextField = new javax.swing.JTextField(); signatureTypeLabel = new javax.swing.JLabel(); - jTextField2 = new javax.swing.JTextField(); + signatureTextField = new javax.swing.JTextField(); offsetLabel = new javax.swing.JLabel(); offsetTextField = new javax.swing.JTextField(); - jButton1 = new javax.swing.JButton(); - jButton2 = new javax.swing.JButton(); - jButton3 = new javax.swing.JButton(); + newTypeButton = new javax.swing.JButton(); + deleteTypeButton = new javax.swing.JButton(); + saveTypeButton = new javax.swing.JButton(); hexPrefixLabel = new javax.swing.JLabel(); signatureTypeComboBox = new javax.swing.JComboBox(); signatureLabel = new javax.swing.JLabel(); @@ -132,17 +145,17 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { org.openide.awt.Mnemonics.setLocalizedText(signatureTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.signatureTypeLabel.text")); // NOI18N - jTextField2.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jTextField2.text")); // NOI18N + signatureTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.signatureTextField.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(offsetLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.offsetLabel.text")); // NOI18N offsetTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.offsetTextField.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(jButton1, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jButton1.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(newTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.newTypeButton.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(jButton2, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jButton2.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(deleteTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.deleteTypeButton.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(jButton3, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jButton3.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(saveTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.saveTypeButton.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(hexPrefixLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.hexPrefixLabel.text")); // NOI18N @@ -170,14 +183,14 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) - .addComponent(jButton1) + .addComponent(newTypeButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(jButton2)) + .addComponent(deleteTypeButton)) .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(37, 37, 37) - .addComponent(jButton3) + .addComponent(saveTypeButton) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addGroup(layout.createSequentialGroup() .addGap(18, 18, 18) @@ -204,7 +217,7 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGap(5, 5, 5) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 84, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(jTextField2, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))) .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() .addGap(2, 2, 2) .addComponent(mimeTypeLabel) @@ -241,7 +254,7 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(jTextField2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(hexPrefixLabel) .addComponent(signatureLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -254,11 +267,11 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { .addGroup(layout.createSequentialGroup() .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(jButton1) - .addComponent(jButton2))) + .addComponent(newTypeButton) + .addComponent(deleteTypeButton))) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(jButton3)))) + .addComponent(saveTypeButton)))) .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); @@ -267,20 +280,20 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel { // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.ButtonGroup buttonGroup1; + private javax.swing.JButton deleteTypeButton; private javax.swing.JLabel hexPrefixLabel; - private javax.swing.JButton jButton1; - private javax.swing.JButton jButton2; - private javax.swing.JButton jButton3; private javax.swing.JScrollPane jScrollPane2; private javax.swing.JSeparator jSeparator1; private javax.swing.JTextArea jTextArea1; - private javax.swing.JTextField jTextField2; private javax.swing.JLabel mimeTypeLabel; private javax.swing.JTextField mimeTypeTextField; + private javax.swing.JButton newTypeButton; private javax.swing.JLabel offsetLabel; private javax.swing.JTextField offsetTextField; private javax.swing.JCheckBox postHitCheckBox; + private javax.swing.JButton saveTypeButton; private javax.swing.JLabel signatureLabel; + private javax.swing.JTextField signatureTextField; private javax.swing.JComboBox signatureTypeComboBox; private javax.swing.JLabel signatureTypeLabel; private javax.swing.JList typesList; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java index 5a5ad78f31..c573b366b9 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -20,39 +20,34 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.util.ArrayList; import java.util.List; +import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException; import org.sleuthkit.datamodel.AbstractFile; /** - * Does file type identification with user-defined file types. + * Does file type identification for user-defined file types. */ final class UserDefinedFileTypeIdentifier { private final List fileTypes; /** - * Creates an object that does file type identification with user-defined - * file types. Does not load the file type definitions. + * Creates an object that can do file type identification for user-defined + * file types. */ UserDefinedFileTypeIdentifier() { this.fileTypes = new ArrayList<>(); + List fileTypeDefs = UserDefinedFileTypesManager.getInstance().getFileTypes(); + this.fileTypes.addAll(fileTypeDefs); } /** - * Loads the set of user-defined file types. - */ - void loadFileTypes() throws UserDefinedFileTypes.UserDefinedFileTypesException { - this.fileTypes.clear(); - this.fileTypes.addAll(UserDefinedFileTypes.getFileTypes()); - } - - /** - * Attempts to identify a file using the set of user-defined file type - * file types. + * Attempts to identify a file using the set of user-defined file type file + * types. * * @param file The file to type. * @return A FileType object or null if identification fails. */ - FileType identify(AbstractFile file) { + FileType identify(final AbstractFile file) { FileType type = null; for (FileType fileType : this.fileTypes) { if (fileType.matches(file)) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java deleted file mode 100644 index 84bc4f8778..0000000000 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.modules.filetypeid; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import javax.xml.parsers.ParserConfigurationException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; -import javax.xml.bind.DatatypeConverter; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.coreutils.XMLUtil; -import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; -import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; - -/** - * Allows a user to define named file types characterized by file signatures. - */ -final class UserDefinedFileTypes { - - private static final Logger logger = Logger.getLogger(UserDefinedFileTypes.class.getName()); - private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; // NON-NLS - private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; // NON-NLS - private static final String AUTOPSY_TYPE_DEFINITIONS_FILE = "AutopsyFileTypeDefinitions.xml"; // NON-NLS - private static final String FILE_TYPES_TAG_NAME = "filetypes"; // NON-NLS - private static final String FILE_TYPE_TAG_NAME = "filetype"; // NON-NLS - private static final String ALERT_ATTRIBUTE = "alert"; // NON-NLS - private static final String TYPE_NAME_TAG_NAME = "typename"; // NON-NLS - private static final String SIGNATURE_TAG_NAME = "signature"; // NON-NLS - private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; // NON-NLS - private static final String BYTES_TAG_NAME = "bytes"; // NON-NLS - private static final String OFFSET_TAG_NAME = "offset"; // NON-NLS - private static final String ENCODING = "UTF-8"; //NON-NLS // RJCTODO: Is this right? - - static { - UserDefinedFileTypes.writePredefinedTypes(); - } - - /** - * Writes the predefined file types definition file. - */ - private static void writePredefinedTypes() { - try { - Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); - String filePathString = filePath.toAbsolutePath().toString(); - List fileTypes = new ArrayList<>(); // RJCTODO - UserDefinedFileTypes.writeFileTypes(fileTypes, filePathString); - } catch (UserDefinedFileTypesException ex) { - // RJCTODO - } - } - - /** - * Gets the user-defined file types. - * - * @return A list of file types, possibly empty - */ - synchronized static List getFileTypes() throws UserDefinedFileTypesException { - Map fileTypes = new HashMap<>(); - UserDefinedFileTypes.readFileTypes(fileTypes, UserDefinedFileTypes.AUTOPSY_TYPE_DEFINITIONS_FILE); - UserDefinedFileTypes.readFileTypes(fileTypes, UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); - return new ArrayList(fileTypes.values()); - } - - /** - * Reads file type definitions from a file into a map of type names to file - * types. File types already loaded into the map will be overwritten by file - * types from the file if the type name is the same. - * - * @param fileTypes The map of type names to file type objects. - * @param fileName The file from which to read the file type definitions. - */ - private static void readFileTypes(Map fileTypes, String fileName) throws UserDefinedFileTypesException { - Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), fileName); - String filePathString = filePath.toAbsolutePath().toString(); - File file = new File(filePathString); - if (file.exists() && file.canRead()) { - for (FileType fileType : UserDefinedFileTypes.XMLReader.readFileTypes(filePathString)) { - fileTypes.put(fileType.getTypeName(), fileType); - } - } - } - - /** - * Sets the user-defined file types. - * - * @param fileTypes The file type definitions. - * @throws UserDefinedFileTypesException - */ - synchronized static void setFileTypes(List fileTypes) throws UserDefinedFileTypesException { - Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.USER_DEFINED_TYPE_DEFINITIONS_FILE); - String filePathString = filePath.toAbsolutePath().toString(); - UserDefinedFileTypes.writeFileTypes(fileTypes, filePathString); - } - - /** - * Writes a set of file type definitions to a given file. - * - * @param fileTypes The file types. - * @param filePaht The destination file. - * @throws UserDefinedFileTypesException - */ - private static void writeFileTypes(List fileTypes, String filePath) throws UserDefinedFileTypesException { - try { - UserDefinedFileTypes.XMLWriter.writeFileTypes(fileTypes, filePath); - } catch (ParserConfigurationException | IOException ex) { - UserDefinedFileTypes.logger.log(Level.SEVERE, "Failed to write file types file", ex); - throw new UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message - } - } - - /** - * Provides a mechanism for writing a set of file type definitions to an XML - * file. - */ - private static class XMLWriter { - - /** - * Writes a set of file type definitions to an XML file. - * - * @param signatures A collection of file types. - * @param filePath The path to the destination file. - */ - private static void writeFileTypes(List fileTypes, String filePath) throws ParserConfigurationException, IOException { - Document doc = XMLUtil.createDoc(); - Element fileTypesElem = doc.createElement(UserDefinedFileTypes.FILE_TYPES_TAG_NAME); - doc.appendChild(fileTypesElem); - for (FileType fileType : fileTypes) { - Element fileTypeElem = UserDefinedFileTypes.XMLWriter.createFileTypeElement(fileType, doc); - fileTypesElem.appendChild(fileTypeElem); - } - if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypes.ENCODING, doc)) { - throw new IOException("Could not save user defined file types"); - } - } - - /** - * Creates an XML representation of a file type. - * - * @param fileType The file type object. - * @param doc The DOM document to use to create the XML. - * @return An XML element. - */ - private static Element createFileTypeElement(FileType fileType, Document doc) { - /** - * Create a file type element with an alert attribute. - */ - Element fileTypeElem = doc.createElement(UserDefinedFileTypes.FILE_TYPE_TAG_NAME); - fileTypeElem.setAttribute(UserDefinedFileTypes.ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); - - /** - * Add a type name child element. - */ - Element typeNameElem = doc.createElement(UserDefinedFileTypes.TYPE_NAME_TAG_NAME); - typeNameElem.setTextContent(fileType.getTypeName()); - fileTypeElem.appendChild(typeNameElem); - - /** - * Add a signature child element with a type attribute. - */ - Signature signature = fileType.getSignature(); - Element signatureElem = doc.createElement(UserDefinedFileTypes.SIGNATURE_TAG_NAME); - signatureElem.setAttribute(UserDefinedFileTypes.SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); - fileTypeElem.appendChild(signatureElem); - - /** - * Add a bytes child element to the signature element. - */ - Element bytesElem = doc.createElement(UserDefinedFileTypes.BYTES_TAG_NAME); - bytesElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); - signatureElem.appendChild(bytesElem); - - /** - * Add an offset child element to the signature element. - */ - Element offsetElem = doc.createElement(UserDefinedFileTypes.OFFSET_TAG_NAME); - offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); - signatureElem.appendChild(offsetElem); - - return fileTypeElem; - } - } - - /** - * Provides a mechanism for reading a set of file type definitions from an - * XML file. - */ - private static class XMLReader { - - /** - * Reads a set of file type definitions from an XML file. - * - * @param filePath The path to the file. - * @return A collection of file types. - */ - private static List readFileTypes(String filePath) throws UserDefinedFileTypesException { - List fileTypes = new ArrayList<>(); - Path xsdPath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypes.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); - String xsdPathString = xsdPath.toAbsolutePath().toString(); - File file = new File(xsdPathString); - if (file.exists() && file.canRead()) { - Document doc = XMLUtil.loadDoc(UserDefinedFileTypes.XMLReader.class, filePath, xsdPathString); - if (doc != null) { - Element fileTypesElem = doc.getDocumentElement(); - if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypes.FILE_TYPES_TAG_NAME)) { - NodeList fileTypeElems = fileTypesElem.getElementsByTagName(UserDefinedFileTypes.FILE_TYPE_TAG_NAME); - for (int i = 0; i < fileTypeElems.getLength(); ++i) { - Element fileTypeElem = (Element) fileTypeElems.item(i); - FileType fileType = UserDefinedFileTypes.XMLReader.parseFileType(fileTypeElem); - fileTypes.add(fileType); - } - } - } - } - return fileTypes; - } - - /** - * Parse a file type definition from a file type XML element. - * - * @param fileTypeElem The XML element. - * @return A file type object. - * @throws UserDefinedFileTypesException - */ - private static FileType parseFileType(Element fileTypeElem) throws UserDefinedFileTypesException { - /** - * Get the alert attribute. - */ - String alertAttribute = fileTypeElem.getAttribute(UserDefinedFileTypes.ALERT_ATTRIBUTE); - boolean alert = Boolean.parseBoolean(alertAttribute); - - /** - * Get the type name child element. - */ - String typeName = UserDefinedFileTypes.getChildElementTextContent(fileTypeElem, UserDefinedFileTypes.TYPE_NAME_TAG_NAME); - - /** - * Get the signature child element. - */ - Element signatureElem; - NodeList signatureElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypes.SIGNATURE_TAG_NAME); - signatureElem = (Element) signatureElems.item(0); - - /** - * Get the signature type attribute from the signature child - * element. - */ - String sigTypeAttribute = signatureElem.getAttribute(typeName); - Signature.Type signatureType = Signature.Type.valueOf(sigTypeAttribute); - - /** - * Get the signature bytes. - */ - String sigBytesString = UserDefinedFileTypes.getChildElementTextContent(signatureElem, UserDefinedFileTypes.TYPE_NAME_TAG_NAME); - byte[] signatureBytes = DatatypeConverter.parseHexBinary(sigBytesString); - - /** - * Get the offset. - */ - String offsetString = UserDefinedFileTypes.getChildElementTextContent(signatureElem, UserDefinedFileTypes.OFFSET_TAG_NAME); - long offset = DatatypeConverter.parseLong(offsetString); - - /** - * Put it all together. - */ - Signature signature = new Signature(signatureBytes, offset, signatureType); - return new FileType(typeName, signature, alert); - } - } - - /** - * Gets the text content of a single child element. - * - * @param elem The parent element. - * @param tagName The tag name of the child element. - * @return The text content. - * @throws UserDefinedFileTypesException - */ - private static String getChildElementTextContent(Element elem, String tagName) throws UserDefinedFileTypesException { - /** - * The checks here are essentially "sanity checks" since the XML was - * already validated using the XSD file. - */ - String textContent = ""; - NodeList childElems = elem.getElementsByTagName(tagName); - if (childElems.getLength() > 0) { - Element childElem = (Element) childElems.item(0); - textContent = childElem.getTextContent(); - if (textContent.isEmpty()) { - UserDefinedFileTypes.logger.log(Level.SEVERE, "File type {0} child element missing text content", tagName); // NON-NLS - } - } else { - UserDefinedFileTypes.logger.log(Level.SEVERE, "File type element missing {0} child element", tagName); // NON-NLS - } - if (textContent.isEmpty()) { - throw new UserDefinedFileTypes.UserDefinedFileTypesException("Error reading file type definitions file, see log for details"); // RJCTODO: Bundle - } - return textContent; - } - - /** - * An exception type to conflate more implementation-details-specific - * exception types (which are logged by this class) into a single generic - * type. - */ - static class UserDefinedFileTypesException extends Exception { - - UserDefinedFileTypesException(String message) { - super(message); - } - } - -} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd index 2512542294..aa8e053534 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd @@ -15,36 +15,35 @@ - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java new file mode 100644 index 0000000000..257e740ecf --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -0,0 +1,511 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.filetypeid; + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import javax.xml.bind.DatatypeConverter; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; +import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; + +/** + * Manages user-defined file types characterized by type names (e.g., MIME type) + * and signatures. + */ +final class UserDefinedFileTypesManager { + + private static final Logger logger = Logger.getLogger(UserDefinedFileTypesManager.class.getName()); + private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; // NON-NLS + private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; // NON-NLS + private static final String PROPERTIES_FILE_NAME = "UserFileTypeDefinitions"; //NON_NLS + private static final String DELETED_PREDEFINED_TYPES_KEY = "DeletedPredefinedTypes"; //NON_NLS + private static final String FILE_TYPES_TAG_NAME = "filetypes"; // NON-NLS + private static final String FILE_TYPE_TAG_NAME = "filetype"; // NON-NLS + private static final String ALERT_ATTRIBUTE = "alert"; // NON-NLS + private static final String TYPE_NAME_TAG_NAME = "typename"; // NON-NLS + private static final String SIGNATURE_TAG_NAME = "signature"; // NON-NLS + private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; // NON-NLS + private static final String BYTES_TAG_NAME = "bytes"; // NON-NLS + private static final String OFFSET_TAG_NAME = "offset"; // NON-NLS + private static final String ENCODING = "UTF-8"; //NON-NLS + private static final String ASCII_ENCODING = "US-ASCII"; //NON-NLS + private static UserDefinedFileTypesManager instance; + private final Map predefinedFileTypes = new HashMap<>(); + private final Map userDefinedFileTypes = new HashMap<>(); + private final Map fileTypes = new HashMap<>(); + + /** + * Gets the user-defined file types manager. + * + * @return A singleton UserDefinedFileTypesManager object. + */ + synchronized static UserDefinedFileTypesManager getInstance() { + if (instance == null) { + instance = new UserDefinedFileTypesManager(); + } + return instance; + } + + /** + * Creates a manager of user-defined file types characterized by type names + * (e.g., MIME type) and signatures. + */ + private UserDefinedFileTypesManager() { + createFileTypesForTesting(); + createStandardPredefinedFileTypes(); + removeDeletedPredefinedFileTypes(); + initializeUserDefinedFileTypes(); + } + + /** + * Adds file types useful for testing this class to the in-memory mapping of + * file type names to predefined file types. + */ + private void createFileTypesForTesting() { + /** + * RJCTODO: This code should be moved into a unit test. + */ + if (true) { + /** + * Create a file type that should match $MBR in Small2 image. + */ + this.predefinedFileTypes.put("predefinedRAW", new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), true)); + + /** + * Create a file type that should match test.txt in the Small2 + * image. + */ + try { + this.predefinedFileTypes.put("predefinedASCII", new FileType("predefinedASCII", new Signature("hello".getBytes(ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true)); + } catch (UnsupportedEncodingException ex) { + logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS + } + } + } + + /** + * Adds standard predefined file types to the in-memory mapping of file type + * names to predefined file types. + */ + private void createStandardPredefinedFileTypes() { + } + + /** + * Removes predefined file types deleted by the user from the in-memory + * mapping of file type names to predefined file types. + */ + private void removeDeletedPredefinedFileTypes() { + String deletedTypes = ModuleSettings.getConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY); + if (null != deletedTypes) { + for (String typeName : deletedTypes.split(", ")) { + this.predefinedFileTypes.remove(typeName); + } + } + } + + /** + * Reads user-defined file types into an in-memory mapping of file type + * names to file types. + */ + private void initializeUserDefinedFileTypes() { + try { + /** + * Read the user-defined types from the backing XML file. + */ + String filePath = getFileTypeDefinitionsFilePath(USER_DEFINED_TYPE_DEFINITIONS_FILE); + File file = new File(filePath); + if (file.exists() && file.canRead()) { + for (FileType fileType : XMLReader.readFileTypes(filePath)) { + userDefinedFileTypes.put(fileType.getTypeName(), fileType); + } + } + + /** + * Combine the predefined and user-defined file types, with + * user-defined file types taking precedence over any predefined + * file types with the same type name. + */ + fileTypes.putAll(predefinedFileTypes); + fileTypes.putAll(userDefinedFileTypes); + + } catch (InvalidXMLException ex) { + // RJCTODO: + } + } + + /** + * Gets the file types. + * + * @return A mapping of file type names to file types, possibly empty. + */ + synchronized List getFileTypes() { + /** + * It is safe to return references to the internal file type objects + * because they are immutable. + */ + return new ArrayList<>(this.fileTypes.values()); + } + + /** + * Adds a new file type definition, overwriting any file type with the same + * type name that already exists. + * + * @param fileType The file type to add. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized void addFileType(FileType fileType) throws UserDefinedFileTypesException { + List newFileTypes = new ArrayList<>(); + newFileTypes.add(fileType); + addFileTypes(newFileTypes); + } + + /** + * Adds a set of new file types, overwriting any file types with the same + * type names that already exist. + * + * @param newFileTypes The file types to add. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized void addFileTypes(Collection newFileTypes) throws UserDefinedFileTypesException { + for (FileType fileType : newFileTypes) { + deleteFromPredefinedTypes(fileType); + userDefinedFileTypes.put(fileType.getTypeName(), fileType); + fileTypes.put(fileType.getTypeName(), fileType); + } + saveUserDefinedTypes(); + } + + /** + * Deletes a file type. + * + * @param fileType The file type to delete. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized void deleteFileType(FileType fileType) throws UserDefinedFileTypesException { + List deletedFileTypes = new ArrayList<>(); + deletedFileTypes.add(fileType); + deleteFileTypes(deletedFileTypes); + } + + /** + * Deletes a set of file types. + * + * @param deletedFileTypes The file types to delete. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized void deleteFileTypes(Collection deletedFileTypes) throws UserDefinedFileTypesException { + for (FileType fileType : deletedFileTypes) { + deleteFromPredefinedTypes(fileType); + userDefinedFileTypes.remove(fileType.getTypeName(), fileType); + fileTypes.remove(fileType.getTypeName(), fileType); + } + saveUserDefinedTypes(); + } + + /** + * Removes a file type, if present, from the in-memory mapping of predefined + * file types and marks the predefined file type as deleted by the user. + * + * @param fileType The file type to delete. + */ + private void deleteFromPredefinedTypes(FileType fileType) { + if (predefinedFileTypes.containsKey(fileType.getTypeName())) { + predefinedFileTypes.remove(fileType.getTypeName()); + String deletedTypesSetting = ModuleSettings.getConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY); + if (null != deletedTypesSetting) { + List deletedTypes = Arrays.asList(deletedTypesSetting.split(", ")); + if (!deletedTypes.contains(fileType.getTypeName())) { + deletedTypesSetting += "," + fileType.getTypeName(); + ModuleSettings.setConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY, deletedTypesSetting); + } + } else { + ModuleSettings.setConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY, fileType.getTypeName()); + } + } + } + + /** + * Persist the user-defined file type definitions. + * + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + private void saveUserDefinedTypes() throws UserDefinedFileTypesException { + String filePath = getFileTypeDefinitionsFilePath(USER_DEFINED_TYPE_DEFINITIONS_FILE); + writeFileTypes(userDefinedFileTypes.values(), filePath); + } + + /** + * Writes a set of file type definitions to a given file. + * + * @param fileTypes The file types. + * @param filePath The destination file. + * @throws UserDefinedFileTypesException + */ + private static void writeFileTypes(Collection fileTypes, String filePath) throws UserDefinedFileTypesException { + try { + XMLWriter.writeFileTypes(fileTypes, filePath); + } catch (ParserConfigurationException | IOException ex) { + UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); + throw new UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message + } + } + + /** + * Gets the absolute path of a file type definitions file. + * + * @param fileName The name of the file. + * @return The absolute path to the file. + */ + private static String getFileTypeDefinitionsFilePath(String fileName) { + Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), fileName); + return filePath.toAbsolutePath().toString(); + } + + /** + * Provides a mechanism for writing a set of file type definitions to an XML + * file. + */ + private static class XMLWriter { + + /** + * Writes a set of file type definitions to an XML file. + * + * @param signatures A collection of file types. + * @param filePath The path to the destination file. + */ + private static void writeFileTypes(Collection fileTypes, String filePath) throws ParserConfigurationException, IOException { + Document doc = XMLUtil.createDoc(); + Element fileTypesElem = doc.createElement(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME); + doc.appendChild(fileTypesElem); + for (FileType fileType : fileTypes) { + Element fileTypeElem = UserDefinedFileTypesManager.XMLWriter.createFileTypeElement(fileType, doc); + fileTypesElem.appendChild(fileTypeElem); + } + if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypesManager.ENCODING, doc)) { + // RJCTODO: If time permits add XMLUtil that properly throws and deprecate this one + throw new IOException("Error saving user defined file types, see log for details"); // NON-NLS + } + } + + /** + * Creates an XML representation of a file type. + * + * @param fileType The file type object. + * @param doc The DOM document to use to create the XML. + * @return An XML element. + */ + private static Element createFileTypeElement(FileType fileType, Document doc) { + /** + * Create a file type element with an alert attribute. + */ + Element fileTypeElem = doc.createElement(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); + fileTypeElem.setAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); + + /** + * Add a type name child element. + */ + Element typeNameElem = doc.createElement(UserDefinedFileTypesManager.TYPE_NAME_TAG_NAME); + typeNameElem.setTextContent(fileType.getTypeName()); + fileTypeElem.appendChild(typeNameElem); + + /** + * Add a signature child element with a type attribute. + */ + Signature signature = fileType.getSignature(); + Element signatureElem = doc.createElement(UserDefinedFileTypesManager.SIGNATURE_TAG_NAME); + signatureElem.setAttribute(UserDefinedFileTypesManager.SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); + fileTypeElem.appendChild(signatureElem); + + /** + * Add a bytes child element to the signature element. + */ + Element bytesElem = doc.createElement(UserDefinedFileTypesManager.BYTES_TAG_NAME); + bytesElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); + signatureElem.appendChild(bytesElem); + + /** + * Add an offset child element to the signature element. + */ + Element offsetElem = doc.createElement(UserDefinedFileTypesManager.OFFSET_TAG_NAME); + offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); + signatureElem.appendChild(offsetElem); + + return fileTypeElem; + } + } + + /** + * Provides a mechanism for reading a set of file type definitions from an + * XML file. + */ + private static class XMLReader { + + /** + * Reads a set of file type definitions from an XML file. + * + * @param filePath The path to the file. + * @return A collection of file types. + */ + private static List readFileTypes(String filePath) throws InvalidXMLException { + List fileTypes = new ArrayList<>(); + Path xsdPath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); + String xsdPathString = xsdPath.toAbsolutePath().toString(); + File file = new File(xsdPathString); + if (file.exists() && file.canRead()) { + Document doc = XMLUtil.loadDoc(UserDefinedFileTypesManager.XMLReader.class, filePath, xsdPathString); + if (doc != null) { + Element fileTypesElem = doc.getDocumentElement(); + if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME)) { + NodeList fileTypeElems = fileTypesElem.getElementsByTagName(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); + for (int i = 0; i < fileTypeElems.getLength(); ++i) { + Element fileTypeElem = (Element) fileTypeElems.item(i); + FileType fileType = UserDefinedFileTypesManager.XMLReader.parseFileType(fileTypeElem); + fileTypes.add(fileType); + } + } + } + } + return fileTypes; + } + + /** + * Parse a file type definition from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return A file type object. + * @throws UserDefinedFileTypesException + */ + private static FileType parseFileType(Element fileTypeElem) throws InvalidXMLException { + /** + * Get the alert attribute. + */ + String alertAttribute = fileTypeElem.getAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE); + boolean alert = Boolean.parseBoolean(alertAttribute); + + /** + * Get the type name child element. + */ + String typeName = UserDefinedFileTypesManager.getChildElementTextContent(fileTypeElem, UserDefinedFileTypesManager.TYPE_NAME_TAG_NAME); + + /** + * Get the signature child element. + */ + Element signatureElem; + NodeList signatureElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypesManager.SIGNATURE_TAG_NAME); + signatureElem = (Element) signatureElems.item(0); + + /** + * Get the signature (interpretation) type attribute from the + * signature child element. + */ + String sigTypeAttribute = signatureElem.getAttribute(UserDefinedFileTypesManager.SIGNATURE_TYPE_ATTRIBUTE); + Signature.Type signatureType = Signature.Type.valueOf(sigTypeAttribute); + + /** + * Get the signature bytes. + */ + String sigBytesString = UserDefinedFileTypesManager.getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.TYPE_NAME_TAG_NAME); + byte[] signatureBytes = DatatypeConverter.parseHexBinary(sigBytesString); + + /** + * Get the offset. + */ + String offsetString = UserDefinedFileTypesManager.getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.OFFSET_TAG_NAME); + long offset = DatatypeConverter.parseLong(offsetString); + + /** + * Put it all together. + */ + Signature signature = new Signature(signatureBytes, offset, signatureType); + return new FileType(typeName, signature, alert); + } + } + + /** + * Gets the text content of a single child element. + * + * @param elem The parent element. + * @param tagName The tag name of the child element. + * @return The text content. + * @throws UserDefinedFileTypesException + */ + private static String getChildElementTextContent(Element elem, String tagName) throws InvalidXMLException { + /** + * The checks here are essentially "sanity checks" since the XML was + * already validated using the XSD file. + */ + NodeList childElems = elem.getElementsByTagName(tagName); + if (childElems.getLength() > 0) { + Element childElem = (Element) childElems.item(0); + String textContent = childElem.getTextContent(); + if (!textContent.isEmpty()) { + return textContent; + } else { + throw new InvalidXMLException("File type " + tagName + " child element missing text content"); // NON-NLS + } + } else { + throw new InvalidXMLException("File type element missing " + tagName + " child element"); // NON-NLS + } + } + + /** + * RJCTODO + */ + static class InvalidXMLException extends Exception { + + InvalidXMLException(String message) { + super(message); + } + } + + /** + * Used to translate more implementation-details-specific exceptions (which + * are logged by this class) into more generic exceptions. + */ + static class UserDefinedFileTypesException extends Exception { + + UserDefinedFileTypesException(String message) { + super(message); + } + } + +} From 4e1da6b73e75a11d0ce19d505b9b0413d85bcfaa Mon Sep 17 00:00:00 2001 From: Eamonn Saunders Date: Mon, 1 Dec 2014 10:21:57 -0500 Subject: [PATCH 08/84] Code that results in UI interaction needs to be invoked on the EDT. --- .../sleuthkit/autopsy/casemodule/Case.java | 76 +++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 6e833bbacb..ba6360fa00 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.casemodule; +import java.awt.EventQueue; import java.awt.Frame; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; @@ -1077,7 +1078,7 @@ public class Case implements SleuthkitCase.ErrorObserver { } //case change helper - private static void doCaseChange(Case toChangeTo) { + private static void doCaseChange(final Case toChangeTo) { logger.log(Level.INFO, "Changing Case to: " + toChangeTo); //NON-NLS if (toChangeTo != null) { // new case is open @@ -1085,39 +1086,49 @@ public class Case implements SleuthkitCase.ErrorObserver { Case.clearTempFolder(); checkSubFolders(toChangeTo); - // enable these menus - CallableSystemAction.get(AddImageAction.class).setEnabled(true); - CallableSystemAction.get(CaseCloseAction.class).setEnabled(true); - CallableSystemAction.get(CasePropertiesAction.class).setEnabled(true); - CallableSystemAction.get(CaseDeleteAction.class).setEnabled(true); // Delete Case menu + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + // enable these menus + CallableSystemAction.get(AddImageAction.class).setEnabled(true); + CallableSystemAction.get(CaseCloseAction.class).setEnabled(true); + CallableSystemAction.get(CasePropertiesAction.class).setEnabled(true); + CallableSystemAction.get(CaseDeleteAction.class).setEnabled(true); // Delete Case menu + + if (toChangeTo.hasData()) { + // open all top components + CoreComponentControl.openCoreWindows(); + } else { + // close all top components + CoreComponentControl.closeCoreWindows(); + } + } + }); - if (toChangeTo.hasData()) { - // open all top components - CoreComponentControl.openCoreWindows(); - } else { - // close all top components - CoreComponentControl.closeCoreWindows(); - } } else { // case is closed - // close all top components first - CoreComponentControl.closeCoreWindows(); + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + // close all top components first + CoreComponentControl.closeCoreWindows(); - // disable these menus - CallableSystemAction.get(AddImageAction.class).setEnabled(false); // Add Image menu - CallableSystemAction.get(CaseCloseAction.class).setEnabled(false); // Case Close menu - CallableSystemAction.get(CasePropertiesAction.class).setEnabled(false); // Case Properties menu - CallableSystemAction.get(CaseDeleteAction.class).setEnabled(false); // Delete Case menu + // disable these menus + CallableSystemAction.get(AddImageAction.class).setEnabled(false); // Add Image menu + CallableSystemAction.get(CaseCloseAction.class).setEnabled(false); // Case Close menu + CallableSystemAction.get(CasePropertiesAction.class).setEnabled(false); // Case Properties menu + CallableSystemAction.get(CaseDeleteAction.class).setEnabled(false); // Delete Case menu - //clear pending notifications - MessageNotifyUtil.Notify.clear(); + //clear pending notifications + MessageNotifyUtil.Notify.clear(); + Frame f = WindowManager.getDefault().getMainWindow(); + f.setTitle(Case.getAppName()); // set the window name to just application name - Frame f = WindowManager.getDefault().getMainWindow(); - f.setTitle(Case.getAppName()); // set the window name to just application name - - //try to force gc to happen - System.gc(); - System.gc(); + //try to force gc to happen + System.gc(); + System.gc(); + } + }); } //log memory usage after case changed @@ -1130,8 +1141,13 @@ public class Case implements SleuthkitCase.ErrorObserver { private static void doCaseNameChange(String newCaseName) { // update case name if (!newCaseName.equals("")) { - Frame f = WindowManager.getDefault().getMainWindow(); - f.setTitle(newCaseName + " - " + Case.getAppName()); // set the window name to the new value + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + Frame f = WindowManager.getDefault().getMainWindow(); + f.setTitle(newCaseName + " - " + Case.getAppName()); // set the window name to the new value + } + }); } } From 39419573170b73c3799744c5852fe271c0de42aa Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Tue, 2 Dec 2014 13:31:27 -0500 Subject: [PATCH 09/84] Reviewed interestingitems for new strings needing localization. Added NOI18N comments in a few places. Removed unused strings from Bundle.properties. --- .../autopsy/modules/interestingitems/Bundle.properties | 4 ---- .../modules/interestingitems/FilesIdentifierIngestModule.java | 2 +- .../modules/interestingitems/InterestingItemDefsManager.java | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties index 68f5433dc1..c15d823cca 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties @@ -4,10 +4,6 @@ OpenIDE-Module-Short-Description=Interesting Files Identifier ingest module. OpenIDE-Module-Name=Interesting Files Identifier InterestingItemsIdentifierIngestModule.moduleName=Interesting Files Identifier InterestingItemsIdentifierIngestModule.moduleDescription=Identifies interesting items as defined by interesting item rule sets. -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title0=Title 1 -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title3=Title 4 -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title2=Title 3 -InterestingFilesIdentifierJobSettingsPanel.filesSetTable.columnModel.title1=Title 2 InterestingItemDefsPanel.newSetButton.text=New Set InterestingItemDefsPanel.editSetButton.text=Edit Set InterestingItemDefsPanel.deleteSetButton.text=Delete Set diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java index 0643a5f722..3b858680f5 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesIdentifierIngestModule.java @@ -109,7 +109,7 @@ final class FilesIdentifierIngestModule implements FileIngestModule { artifact.addAttribute(ruleNameAttribute); } catch (TskCoreException ex) { - FilesIdentifierIngestModule.logger.log(Level.SEVERE, "Error posting to the blackboard", ex); + FilesIdentifierIngestModule.logger.log(Level.SEVERE, "Error posting to the blackboard", ex); //NOI18N } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java index 2f1c31caba..735683cc58 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsManager.java @@ -255,11 +255,11 @@ final class InterestingItemDefsManager extends Observable { if (!rules.containsKey(rule.getName())) { rules.put(rule.getName(), rule); } else { - logger.log(Level.SEVERE, "Found duplicate rule {0} for set named {1} in interesting file sets definition file at {2}, discarding malformed set", new Object[]{rule.getName(), setName, filePath}); + logger.log(Level.SEVERE, "Found duplicate rule {0} for set named {1} in interesting file sets definition file at {2}, discarding malformed set", new Object[]{rule.getName(), setName, filePath}); //NOI18N return; } } else { - logger.log(Level.SEVERE, "Found malformed rule for set named {0} in interesting file sets definition file at {1}, discarding malformed set", new Object[]{setName, filePath}); + logger.log(Level.SEVERE, "Found malformed rule for set named {0} in interesting file sets definition file at {1}, discarding malformed set", new Object[]{setName, filePath}); //NOI18N return; } } From 5da64cc76e5d94461bc4ff359b3a681f8acb3ebb Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 2 Dec 2014 13:42:13 -0500 Subject: [PATCH 10/84] Make predefined user types hidden, not deletable (part 1) --- .../UserDefinedFileTypeIdentifier.java | 1 - .../UserDefinedFileTypesManager.java | 133 +++++++----------- 2 files changed, 52 insertions(+), 82 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java index c573b366b9..f7dac840ca 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.util.ArrayList; import java.util.List; -import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException; import org.sleuthkit.datamodel.AbstractFile; /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 257e740ecf..ee0f601bf2 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -24,7 +24,6 @@ import java.io.UnsupportedEncodingException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -36,7 +35,6 @@ import org.w3c.dom.Element; import org.w3c.dom.NodeList; import javax.xml.bind.DatatypeConverter; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.XMLUtil; import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; @@ -51,8 +49,6 @@ final class UserDefinedFileTypesManager { private static final Logger logger = Logger.getLogger(UserDefinedFileTypesManager.class.getName()); private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; // NON-NLS private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; // NON-NLS - private static final String PROPERTIES_FILE_NAME = "UserFileTypeDefinitions"; //NON_NLS - private static final String DELETED_PREDEFINED_TYPES_KEY = "DeletedPredefinedTypes"; //NON_NLS private static final String FILE_TYPES_TAG_NAME = "filetypes"; // NON-NLS private static final String FILE_TYPE_TAG_NAME = "filetype"; // NON-NLS private static final String ALERT_ATTRIBUTE = "alert"; // NON-NLS @@ -71,7 +67,7 @@ final class UserDefinedFileTypesManager { /** * Gets the user-defined file types manager. * - * @return A singleton UserDefinedFileTypesManager object. + * @return A singleton user-defined file types manager. */ synchronized static UserDefinedFileTypesManager getInstance() { if (instance == null) { @@ -85,55 +81,57 @@ final class UserDefinedFileTypesManager { * (e.g., MIME type) and signatures. */ private UserDefinedFileTypesManager() { - createFileTypesForTesting(); - createStandardPredefinedFileTypes(); - removeDeletedPredefinedFileTypes(); - initializeUserDefinedFileTypes(); - } - - /** - * Adds file types useful for testing this class to the in-memory mapping of - * file type names to predefined file types. - */ - private void createFileTypesForTesting() { - /** - * RJCTODO: This code should be moved into a unit test. - */ - if (true) { - /** - * Create a file type that should match $MBR in Small2 image. - */ - this.predefinedFileTypes.put("predefinedRAW", new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), true)); - - /** - * Create a file type that should match test.txt in the Small2 - * image. - */ - try { - this.predefinedFileTypes.put("predefinedASCII", new FileType("predefinedASCII", new Signature("hello".getBytes(ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true)); - } catch (UnsupportedEncodingException ex) { - logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS - } - } + loadPredefinedFileTypes(); + loadUserDefinedFileTypes(); } /** * Adds standard predefined file types to the in-memory mapping of file type * names to predefined file types. */ - private void createStandardPredefinedFileTypes() { - } + private void loadPredefinedFileTypes() { + // RJCTODO: Remove + /** + * Create a file type that should match $MBR in Small2 image. + */ + this.fileTypes.put("predefinedRAW", new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), true)); - /** - * Removes predefined file types deleted by the user from the in-memory - * mapping of file type names to predefined file types. - */ - private void removeDeletedPredefinedFileTypes() { - String deletedTypes = ModuleSettings.getConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY); - if (null != deletedTypes) { - for (String typeName : deletedTypes.split(", ")) { - this.predefinedFileTypes.remove(typeName); - } + /** + * Create a file type that should match test.txt in the Small2 image. + */ + // RJCTODO: Remove + try { + this.fileTypes.put("predefinedASCII", new FileType("predefinedASCII", new Signature("hello".getBytes(ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true)); + } catch (UnsupportedEncodingException ex) { + logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS + } + + try { + // RJCTODO: Remove this code from TikaFileTypeDetector.java +// try { +// byte buf[]; +// int len = abstractFile.read(buffer, 0, BUFFER_SIZE); +// if (len < BUFFER_SIZE) { +// buf = new byte[len]; +// System.arraycopy(buffer, 0, buf, 0, len); +// } else { +// buf = buffer; +// } +// +// // the xml detection in Tika tries to parse the entire file and throws exceptions +// // for files that are not valid XML +// try { +// String tagHeader = new String(buf, 0, 5); +// if (tagHeader.equals(" newFileTypes) throws UserDefinedFileTypesException { for (FileType fileType : newFileTypes) { - deleteFromPredefinedTypes(fileType); userDefinedFileTypes.put(fileType.getTypeName(), fileType); fileTypes.put(fileType.getTypeName(), fileType); } @@ -233,35 +227,12 @@ final class UserDefinedFileTypesManager { */ synchronized void deleteFileTypes(Collection deletedFileTypes) throws UserDefinedFileTypesException { for (FileType fileType : deletedFileTypes) { - deleteFromPredefinedTypes(fileType); userDefinedFileTypes.remove(fileType.getTypeName(), fileType); fileTypes.remove(fileType.getTypeName(), fileType); } saveUserDefinedTypes(); } - /** - * Removes a file type, if present, from the in-memory mapping of predefined - * file types and marks the predefined file type as deleted by the user. - * - * @param fileType The file type to delete. - */ - private void deleteFromPredefinedTypes(FileType fileType) { - if (predefinedFileTypes.containsKey(fileType.getTypeName())) { - predefinedFileTypes.remove(fileType.getTypeName()); - String deletedTypesSetting = ModuleSettings.getConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY); - if (null != deletedTypesSetting) { - List deletedTypes = Arrays.asList(deletedTypesSetting.split(", ")); - if (!deletedTypes.contains(fileType.getTypeName())) { - deletedTypesSetting += "," + fileType.getTypeName(); - ModuleSettings.setConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY, deletedTypesSetting); - } - } else { - ModuleSettings.setConfigSetting(PROPERTIES_FILE_NAME, DELETED_PREDEFINED_TYPES_KEY, fileType.getTypeName()); - } - } - } - /** * Persist the user-defined file type definitions. * From 2a851dd267423a043c5f3d83a0192ee7fe33519c Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Tue, 2 Dec 2014 14:28:48 -0500 Subject: [PATCH 11/84] Marked internal/log/exception strings with NOI18N. Created Bundle_ja.properties. --- .../modules/android/AndroidIngestModule.java | 26 ++++++------- .../android/BrowserLocationAnalyzer.java | 24 ++++++------ .../modules/android/Bundle_ja.properties | 0 .../android/CacheLocationAnalyzer.java | 8 ++-- .../modules/android/CallLogAnalyzer.java | 32 ++++++++-------- .../modules/android/ContactAnalyzer.java | 38 +++++++++---------- .../android/GoogleMapLocationAnalyzer.java | 32 ++++++++-------- .../modules/android/TangoMessageAnalyzer.java | 32 ++++++++-------- .../modules/android/TextMessageAnalyzer.java | 30 +++++++-------- .../modules/android/WWFMessageAnalyzer.java | 22 +++++------ 10 files changed, 122 insertions(+), 122 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java index f3d1e56019..ee0dac4221 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java @@ -57,7 +57,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Contacts"); + errors.add("Error getting Contacts"); //NOI18N } try { @@ -67,7 +67,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Call Logs"); + errors.add("Error getting Call Logs"); //NOI18N } try { @@ -77,7 +77,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Text Messages"); + errors.add("Error getting Text Messages"); //NOI18N } try { @@ -87,7 +87,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Tango Messages"); + errors.add("Error getting Tango Messages"); //NOI18N } try { @@ -97,7 +97,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Words with Friends Messages"); + errors.add("Error getting Words with Friends Messages"); //NOI18N } try { @@ -107,7 +107,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Google Map Locations"); + errors.add("Error getting Google Map Locations"); //NOI18N } try { @@ -117,14 +117,14 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Browser Locations"); + errors.add("Error getting Browser Locations"); //NOI18N } try { CacheLocationAnalyzer.findGeoLocations(); progressBar.progress(8); } catch (Exception e) { - errors.add("Error getting Cache Locations"); + errors.add("Error getting Cache Locations"); //NOI18N } // create the final message for inbox @@ -133,20 +133,20 @@ class AndroidIngestModule implements DataSourceIngestModule { IngestMessage.MessageType msgLevel = IngestMessage.MessageType.INFO; if (errors.isEmpty() == false) { msgLevel = IngestMessage.MessageType.ERROR; - errorMessage.append("Errors were encountered"); + errorMessage.append("Errors were encountered"); //NOI18N for (String msg : errors) { errorMessage.append("
  • ").append(msg).append("
    • \n"); //NON-NLS } errorMessage.append("\n"); //NON-NLS if (errors.size() == 1) { - errorMsgSubject = "One error was found"; + errorMsgSubject = "One error was found"; //NOI18N } else { - errorMsgSubject = "errors found: " + errors.size(); + errorMsgSubject = "errors found: " + errors.size(); //NOI18N } } else { - errorMessage.append("No errors"); - errorMsgSubject = "No errors"; + errorMessage.append("No errors"); //NOI18N + errorMsgSubject = "No errors"; //NOI18N } services.postMessage(IngestMessage.createMessage(msgLevel, AndroidModuleFactory.getModuleName(), "Finished Analysis: " + errorMsgSubject, errorMessage.toString())); diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java index 69add3d978..acf6774a6f 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java @@ -43,7 +43,7 @@ class BrowserLocationAnalyzer { public static void findGeoLocations() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List abstractFiles = skCase.findAllFilesWhere("name LIKE 'CachedGeoposition%.db'"); //get exact file names + List abstractFiles = skCase.findAllFilesWhere("name LIKE 'CachedGeoposition%.db'"); //NOI18N //get exact file names for (AbstractFile abstractFile : abstractFiles) { try { @@ -54,11 +54,11 @@ class BrowserLocationAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findGeoLocationsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Browser Location files", e); + logger.log(Level.SEVERE, "Error parsing Browser Location files", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Browser Location files", e); + logger.log(Level.SEVERE, "Error finding Browser Location files", e); //NOI18N } } @@ -71,22 +71,22 @@ class BrowserLocationAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error connecting to sql database", e); + logger.log(Level.SEVERE, "Error connecting to sql database", e); //NOI18N return; } try { resultSet = statement.executeQuery( - "Select timestamp, latitude, longitude, accuracy FROM CachedPosition;"); + "Select timestamp, latitude, longitude, accuracy FROM CachedPosition;"); //NOI18N while (resultSet.next()) { - Long timestamp = Long.valueOf(resultSet.getString("timestamp")) / 1000; - double latitude = Double.valueOf(resultSet.getString("latitude")); - double longitude = Double.valueOf(resultSet.getString("longitude")); + Long timestamp = Long.valueOf(resultSet.getString("timestamp")) / 1000; //NOI18N + double latitude = Double.valueOf(resultSet.getString("latitude")); //NOI18N + double longitude = Double.valueOf(resultSet.getString("longitude")); //NOI18N BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), moduleName, latitude)); @@ -96,7 +96,7 @@ class BrowserLocationAnalyzer { // bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VALUE.getTypeID(),moduleName, accuracy)); } } catch (Exception e) { - logger.log(Level.SEVERE, "Error Putting artifacts to Blackboard", e); + logger.log(Level.SEVERE, "Error Putting artifacts to Blackboard", e); //NOI18N } finally { try { if (resultSet != null) { @@ -105,7 +105,7 @@ class BrowserLocationAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NOI18N } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java index 89d5472f25..6e5bae3027 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java @@ -43,7 +43,7 @@ class CacheLocationAnalyzer { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List abstractFiles = skCase.findAllFilesWhere("name ='cache.cell' OR name='cache.wifi'"); //get exact file names + List abstractFiles = skCase.findAllFilesWhere("name ='cache.cell' OR name='cache.wifi'"); //NOI18N //get exact file names for (AbstractFile abstractFile : abstractFiles) { try { @@ -55,11 +55,11 @@ class CacheLocationAnalyzer { findGeoLocationsInFile(jFile, abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing cached Location files", e); + logger.log(Level.SEVERE, "Error parsing cached Location files", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding cached Location files", e); + logger.log(Level.SEVERE, "Error finding cached Location files", e); //NOI18N } } @@ -124,7 +124,7 @@ class CacheLocationAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Cached GPS locations to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Cached GPS locations to Blackboard", e); //NOI18N } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java index 5f06532860..b213e95004 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java @@ -47,11 +47,11 @@ class CallLogAnalyzer { /** the where clause(without 'where' of sql select statement to choose call * log dbs, update the list of file names to include more files */ - private static final String fileNameQuery = Stream.of("'logs.db'", "'contacts2.db'", "'contacts.db'") - .collect(Collectors.joining(" OR name = ", "name = ", "")); + private static final String fileNameQuery = Stream.of("'logs.db'", "'contacts2.db'", "'contacts.db'") //NOI18N + .collect(Collectors.joining(" OR name = ", "name = ", "")); //NOI18N /** the names of tables that potentially hold call logs in the dbs */ - private static final Iterable tableNames = Arrays.asList("calls", "logs"); + private static final Iterable tableNames = Arrays.asList("calls", "logs"); //NOI18N public static void findCallLogs() { try { @@ -63,11 +63,11 @@ class CallLogAnalyzer { ContentUtils.writeToFile(abstractFile, file); findCallLogsInDB(file.toString(), abstractFile); } catch (IOException e) { - logger.log(Level.SEVERE, "Error writing temporary call log db to disk", e); + logger.log(Level.SEVERE, "Error writing temporary call log db to disk", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding call logs", e); + logger.log(Level.SEVERE, "Error finding call logs", e); //NOI18N } } @@ -76,20 +76,20 @@ class CallLogAnalyzer { if (DatabasePath == null || DatabasePath.isEmpty()) { return; } - try (Connection connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + try (Connection connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N Statement statement = connection.createStatement();) { for (String tableName : tableNames) { try (ResultSet resultSet = statement.executeQuery( - "SELECT number,date,duration,type, name FROM " + tableName + " ORDER BY date DESC;");) { - logger.log(Level.INFO, "Reading call log from table {0} in db {1}", new Object[]{tableName, DatabasePath}); + "SELECT number,date,duration,type, name FROM " + tableName + " ORDER BY date DESC;");) { //NOI18N + logger.log(Level.INFO, "Reading call log from table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NOI18N while (resultSet.next()) { Long date = resultSet.getLong("date") / 1000; - final CallDirection direction = CallDirection.fromType(resultSet.getInt("type")); + final CallDirection direction = CallDirection.fromType(resultSet.getInt("type")); //NOI18N String directionString = direction != null ? direction.getDisplayName() : ""; - final String number = resultSet.getString("number"); - final long duration = resultSet.getLong("duration");//duration of call is in seconds - final String name = resultSet.getString("name");// name of person dialed or called. null if unregistered + final String number = resultSet.getString("number"); //NOI18N + final long duration = resultSet.getLong("duration"); //NOI18N //duration of call is in seconds + final String name = resultSet.getString("name"); //NOI18N // name of person dialed or called. null if unregistered try { BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); //create a call log and then add attributes from result set. @@ -104,21 +104,21 @@ class CallLogAnalyzer { bba.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION.getTypeID(), moduleName, directionString)); bba.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error posting call log record to the Blackboard", ex); + logger.log(Level.SEVERE, "Error posting call log record to the Blackboard", ex); //NOI18N } } } catch (SQLException e) { - logger.log(Level.WARNING, "Could not read table {0} in db {1}", new Object[]{tableName, DatabasePath}); + logger.log(Level.WARNING, "Could not read table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NOI18N } } } catch (SQLException e) { - logger.log(Level.SEVERE, "Could not parse call log; error connecting to db " + DatabasePath, e); + logger.log(Level.SEVERE, "Could not parse call log; error connecting to db " + DatabasePath, e); //NOI18N } } private static enum CallDirection { - INCOMING(1, "Incoming"), OUTGOING(2, "Outgoing"), MISSED(3, "Missed"); + INCOMING(1, "Incoming"), OUTGOING(2, "Outgoing"), MISSED(3, "Missed"); //NOI18N private final int type; diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java index 173d506372..68e021f0fa 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java @@ -45,7 +45,7 @@ class ContactAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NOI18N //get exact file names if (absFiles.isEmpty()) { return; } @@ -55,11 +55,11 @@ class ContactAnalyzer { ContentUtils.writeToFile(AF, jFile); findContactsInDB(jFile.toString(), AF); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts", e); + logger.log(Level.SEVERE, "Error parsing Contacts", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Contacts", e); + logger.log(Level.SEVERE, "Error finding Contacts", e); //NOI18N } } @@ -78,11 +78,11 @@ class ContactAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N return; } @@ -90,13 +90,13 @@ class ContactAnalyzer { // get display_name, mimetype(email or phone number) and data1 (phonenumber or email address depending on mimetype) //sorted by name, so phonenumber/email would be consecutive for a person if they exist. resultSet = statement.executeQuery( - "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" - + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" - + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " - + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" - + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" - + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" - + "ORDER BY name_raw_contact.display_name ASC;"); + "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NOI18N + + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NOI18N + + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NOI18N + + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NOI18N + + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NOI18N + + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NOI18N + + "ORDER BY name_raw_contact.display_name ASC;"); //NOI18N BlackboardArtifact bba; bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); @@ -105,15 +105,15 @@ class ContactAnalyzer { String mimetype; // either phone or email String data1; // the phone number or email while (resultSet.next()) { - name = resultSet.getString("display_name"); - data1 = resultSet.getString("data1"); - mimetype = resultSet.getString("mimetype"); + name = resultSet.getString("display_name"); //NOI18N + data1 = resultSet.getString("data1"); //NOI18N + mimetype = resultSet.getString("mimetype"); //NOI18N // System.out.println(resultSet.getString("data1") + resultSet.getString("mimetype") + resultSet.getString("display_name")); //Test code if (name.equals(oldName) == false) { bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } - if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { + if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NOI18N bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getTypeID(), moduleName, data1)); } else { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getTypeID(), moduleName, data1)); @@ -122,7 +122,7 @@ class ContactAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NOI18N } finally { try { if (resultSet != null) { @@ -131,7 +131,7 @@ class ContactAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NOI18N } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java index 0174b81e1f..09925e3a8f 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java @@ -44,7 +44,7 @@ class GoogleMapLocationAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='da_destination_history'"); //get exact file name + absFiles = skCase.findAllFilesWhere("name ='da_destination_history'"); //NOI18N //get exact file name if (absFiles.isEmpty()) { return; } @@ -54,11 +54,11 @@ class GoogleMapLocationAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findGeoLocationsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Google map locations", e); + logger.log(Level.SEVERE, "Error parsing Google map locations", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Google map locations", e); + logger.log(Level.SEVERE, "Error finding Google map locations", e); //NOI18N } } @@ -71,27 +71,27 @@ class GoogleMapLocationAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N return; } try { resultSet = statement.executeQuery( - "Select time,dest_lat,dest_lng,dest_title,dest_address,source_lat,source_lng FROM destination_history;"); + "Select time,dest_lat,dest_lng,dest_title,dest_address,source_lat,source_lng FROM destination_history;"); //NOI18N while (resultSet.next()) { - Long time = Long.valueOf(resultSet.getString("time")) / 1000; - String dest_title = resultSet.getString("dest_title"); - String dest_address = resultSet.getString("dest_address"); + Long time = Long.valueOf(resultSet.getString("time")) / 1000; //NOI18N + String dest_title = resultSet.getString("dest_title"); //NOI18N + String dest_address = resultSet.getString("dest_address"); //NOI18N - double dest_lat = convertGeo(resultSet.getString("dest_lat")); - double dest_lng = convertGeo(resultSet.getString("dest_lng")); - double source_lat = convertGeo(resultSet.getString("source_lat")); - double source_lng = convertGeo(resultSet.getString("source_lng")); + double dest_lat = convertGeo(resultSet.getString("dest_lat")); //NOI18N + double dest_lng = convertGeo(resultSet.getString("dest_lng")); //NOI18N + double source_lat = convertGeo(resultSet.getString("source_lat")); //NOI18N + double source_lng = convertGeo(resultSet.getString("source_lng")); //NOI18N // bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT);//src @@ -123,7 +123,7 @@ class GoogleMapLocationAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Google map locations to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Google map locations to the Blackboard", e); //NOI18N } finally { try { if (resultSet != null) { @@ -132,7 +132,7 @@ class GoogleMapLocationAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing the database", e); + logger.log(Level.SEVERE, "Error closing the database", e); //NOI18N } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java index 23e4f07695..5b8cee3b4d 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java @@ -45,18 +45,18 @@ class TangoMessageAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='tc.db' "); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='tc.db' "); //NOI18N //get exact file names for (AbstractFile abstractFile : absFiles) { try { File jFile = new File(Case.getCurrentCase().getTempDirectory(), abstractFile.getName()); ContentUtils.writeToFile(abstractFile, jFile); findTangoMessagesInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Tango messages", e); + logger.log(Level.SEVERE, "Error parsing Tango messages", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Tango messages", e); + logger.log(Level.SEVERE, "Error finding Tango messages", e); //NOI18N } } @@ -69,31 +69,31 @@ class TangoMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N return; } try { resultSet = statement.executeQuery( - "Select conv_id, create_time,direction,payload FROM messages ORDER BY create_time DESC;"); + "Select conv_id, create_time,direction,payload FROM messages ORDER BY create_time DESC;"); //NOI18N String conv_id; // seems to wrap around the message found in payload after decoding from base-64 String direction; // 1 incoming, 2 outgoing String payload; // seems to be a base64 message wrapped by the conv_id while (resultSet.next()) { - conv_id = resultSet.getString("conv_id"); - Long create_time = Long.valueOf(resultSet.getString("create_time")) / 1000; - if (resultSet.getString("direction").equals("1")) { - direction = "Incoming"; + conv_id = resultSet.getString("conv_id"); //NOI18N + Long create_time = Long.valueOf(resultSet.getString("create_time")) / 1000; //NOI18N + if (resultSet.getString("direction").equals("1")) { //NOI18N + direction = "Incoming"; //NOI18N } else { - direction = "Outgoing"; + direction = "Outgoing"; //NOI18N } - payload = resultSet.getString("payload"); + payload = resultSet.getString("payload"); //NOI18N BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create a call log and then add attributes from result set. bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), moduleName, create_time)); @@ -104,7 +104,7 @@ class TangoMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Tango messages to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Tango messages to the Blackboard", e); //NOI18N } finally { try { if (resultSet != null) { @@ -113,7 +113,7 @@ class TangoMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NOI18N } } } @@ -126,7 +126,7 @@ class TangoMessageAnalyzer { String Z = new String(decoded, "UTF-8"); result = Z.split(wrapper)[1]; } catch (Exception e) { - logger.log(Level.SEVERE, "Error decoding a Tango message", e); + logger.log(Level.SEVERE, "Error decoding a Tango message", e); //NOI18N } return result; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java index 627c64f327..0b58b75313 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java @@ -43,7 +43,7 @@ class TextMessageAnalyzer { public static void findTexts() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //get exact file name + List absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NOI18N //get exact file name for (AbstractFile abstractFile : absFiles) { try { @@ -51,11 +51,11 @@ class TextMessageAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findTextsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages", e); + logger.log(Level.SEVERE, "Error parsing text messages", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding text messages", e); + logger.log(Level.SEVERE, "Error finding text messages", e); //NOI18N } } @@ -68,17 +68,17 @@ class TextMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N return; } try { resultSet = statement.executeQuery( - "Select address,date,read,type,subject,body FROM sms;"); + "Select address,date,read,type,subject,body FROM sms;"); //NOI18N String address; // may be phone number, or other addresses @@ -87,15 +87,15 @@ class TextMessageAnalyzer { Integer read; // may be unread = 0, read = 1 String body; //message body while (resultSet.next()) { - address = resultSet.getString("address"); - Long date = Long.valueOf(resultSet.getString("date")) / 1000; + address = resultSet.getString("address"); //NOI18N + Long date = Long.valueOf(resultSet.getString("date")) / 1000; //NOI18N - read = resultSet.getInt("read"); - subject = resultSet.getString("subject"); - body = resultSet.getString("body"); + read = resultSet.getInt("read"); //NOI18N + subject = resultSet.getString("subject"); //NOI18N + body = resultSet.getString("body"); //NOI18N BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create Message artifact and then add attributes from result set. - if (resultSet.getString("type").equals("1")) { + if (resultSet.getString("type").equals("1")) { //NOI18N bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION.getTypeID(), moduleName, "Incoming")); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM.getTypeID(), moduleName, address)); } else { @@ -111,7 +111,7 @@ class TextMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NOI18N } finally { try { if (resultSet != null) { @@ -120,7 +120,7 @@ class TextMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NOI18N } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java index 3bda358a24..d5c584dbce 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java @@ -44,7 +44,7 @@ class WWFMessageAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='WordsFramework' "); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='WordsFramework' "); //NOI18N //get exact file names for (AbstractFile abstractFile : absFiles) { try { @@ -53,11 +53,11 @@ class WWFMessageAnalyzer { findWWFMessagesInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing WWF messages", e); + logger.log(Level.SEVERE, "Error parsing WWF messages", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding WWF messages", e); + logger.log(Level.SEVERE, "Error finding WWF messages", e); //NOI18N } } @@ -74,23 +74,23 @@ class WWFMessageAnalyzer { connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N return; } try { resultSet = statement.executeQuery( - "SELECT message,strftime('%s' ,created_at) as datetime,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); + "SELECT message,strftime('%s' ,created_at) as datetime,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); //NOI18N String message; // WWF Message String user_id; // the ID of the user who sent the message. String game_id; // ID of the game which the the message was sent. while (resultSet.next()) { - message = resultSet.getString("message"); - Long created_at = resultSet.getLong("datetime"); - user_id = resultSet.getString("user_id"); - game_id = resultSet.getString("game_id"); + message = resultSet.getString("message"); //NOI18N + Long created_at = resultSet.getLong("datetime"); //NOI18N + user_id = resultSet.getString("user_id"); //NOI18N + game_id = resultSet.getString("game_id"); //NOI18N BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create a call log and then add attributes from result set. bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), moduleName, created_at)); @@ -100,7 +100,7 @@ class WWFMessageAnalyzer { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MESSAGE_TYPE.getTypeID(), moduleName, "Words With Friends Message")); } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing WWF messages to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing WWF messages to the Blackboard", e); //NOI18N } finally { try { if (resultSet != null) { @@ -109,7 +109,7 @@ class WWFMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NOI18N } } } From 5751a4aa8ba3be372ff690f1777121f38c859e46 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Tue, 2 Dec 2014 14:56:03 -0500 Subject: [PATCH 12/84] Marked internal/log/exception strings with NOI18N. Created Bundle_ja.properties. --- .../autopsy/modules/iOS/Bundle_ja.properties | 0 .../autopsy/modules/iOS/CallLogAnalyzer.java | 32 +++++++------- .../autopsy/modules/iOS/ContactAnalyzer.java | 44 +++++++++---------- .../modules/iOS/TextMessageAnalyzer.java | 30 ++++++------- 4 files changed, 53 insertions(+), 53 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java index 38831120e0..3514a5d9bc 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java @@ -49,7 +49,7 @@ class CallLogAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //get exact file names + absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NOI18N //get exact file names if (absFiles.isEmpty()) { return; } @@ -61,11 +61,11 @@ class CallLogAnalyzer { fileId = AF.getId(); findCallLogsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs", e); + logger.log(Level.SEVERE, "Error parsing Call logs", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Call logs", e); + logger.log(Level.SEVERE, "Error finding Call logs", e); //NOI18N } } @@ -74,11 +74,11 @@ class CallLogAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N } Case currentCase = Case.getCurrentCase(); @@ -87,7 +87,7 @@ class CallLogAnalyzer { AbstractFile f = skCase.getAbstractFileById(fId); try { resultSet = statement.executeQuery( - "SELECT number,date,duration,type, name FROM calls ORDER BY date DESC;"); + "SELECT number,date,duration,type, name FROM calls ORDER BY date DESC;"); //NOI18N BlackboardArtifact bba; String name; // name of person dialed or called. null if unregistered @@ -97,14 +97,14 @@ class CallLogAnalyzer { String type; // 1 incoming, 2 outgoing, 3 missed while (resultSet.next()) { - name = resultSet.getString("name"); - number = resultSet.getString("number"); - duration = resultSet.getString("duration"); - date = resultSet.getString("date"); - type = resultSet.getString("type"); + name = resultSet.getString("name"); //NOI18N + number = resultSet.getString("number"); //NOI18N + duration = resultSet.getString("duration"); //NOI18N + date = resultSet.getString("date"); //NOI18N + type = resultSet.getString("type"); //NOI18N bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); //create a call log and then add attributes from result set. - if(type.equalsIgnoreCase("outgoing")) { + if(type.equalsIgnoreCase("outgoing")) { //NOI18N bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO.getTypeID(), moduleName, number)); } else { /// Covers INCOMING and MISSED @@ -117,18 +117,18 @@ class CallLogAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NOI18N } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing the database", e); + logger.log(Level.SEVERE, "Error closing the database", e); //NOI18N } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NOI18N } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java index 0e7ee51e76..a119666993 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java @@ -56,7 +56,7 @@ class ContactAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name LIKE '%call_history%' "); //get exact file names + absFiles = skCase.findAllFilesWhere("name LIKE '%call_history%' "); //NOI18N //get exact file names if (absFiles.isEmpty()) { //asdfkjasfakljsdfhlaksdjfhasdlkjf return; } @@ -71,11 +71,11 @@ class ContactAnalyzer { fileId = AF.getId(); //findContactsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts", e); + logger.log(Level.SEVERE, "Error parsing Contacts", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Contacts", e); + logger.log(Level.SEVERE, "Error finding Contacts", e); //NOI18N } } @@ -90,11 +90,11 @@ class ContactAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N } Case currentCase = Case.getCurrentCase(); @@ -105,13 +105,13 @@ class ContactAnalyzer { // get display_name, mimetype(email or phone number) and data1 (phonenumber or email address depending on mimetype) //sorted by name, so phonenumber/email would be consecutive for a person if they exist. resultSet = statement.executeQuery( - "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" - + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" - + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " - + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" - + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" - + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" - + "ORDER BY name_raw_contact.display_name ASC;"); + "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NOI18N + + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NOI18N + + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NOI18N + + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NOI18N + + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NOI18N + + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NOI18N + + "ORDER BY name_raw_contact.display_name ASC;"); //NOI18N BlackboardArtifact bba; bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); @@ -120,15 +120,15 @@ class ContactAnalyzer { String mimetype; // either phone or email String data1; // the phone number or email while (resultSet.next()) { - name = resultSet.getString("display_name"); - data1 = resultSet.getString("data1"); - mimetype = resultSet.getString("mimetype"); + name = resultSet.getString("display_name"); //NOI18N + data1 = resultSet.getString("data1"); //NOI18N + mimetype = resultSet.getString("mimetype"); //NOI18N // System.out.println(resultSet.getString("data1") + resultSet.getString("mimetype") + resultSet.getString("display_name")); //Test code if (name.equals(oldName) == false) { bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } - if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { + if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NOI18N bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getTypeID(), moduleName, data1)); } else { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getTypeID(), moduleName, data1)); @@ -137,18 +137,18 @@ class ContactAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NOI18N } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NOI18N } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NOI18N } } @@ -186,13 +186,13 @@ class ContactAnalyzer { ostream.write(c); } } catch (IOException e) { - System.out.println("Error: " + e.getMessage()); + System.out.println("Error: " + e.getMessage()); //NOI18N } finally { try { istream.close(); ostream.close(); } catch (IOException e) { - System.out.println("File did not close"); + System.out.println("File did not close"); //NOI18N } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java index 434bcd0ea0..18dae2c61c 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java @@ -49,7 +49,7 @@ class TextMessageAnalyzer { void findTexts() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //get exact file name + absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NOI18N //get exact file name if (absFiles.isEmpty()) { return; } @@ -61,11 +61,11 @@ class TextMessageAnalyzer { fileId = AF.getId(); findTextsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages", e); + logger.log(Level.SEVERE, "Error parsing text messages", e); //NOI18N } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding text messages", e); + logger.log(Level.SEVERE, "Error finding text messages", e); //NOI18N } } @@ -74,11 +74,11 @@ class TextMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); + Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); + logger.log(Level.SEVERE, "Error opening database", e); //NOI18N } Case currentCase = Case.getCurrentCase(); @@ -87,7 +87,7 @@ class TextMessageAnalyzer { AbstractFile f = skCase.getAbstractFileById(fId); try { resultSet = statement.executeQuery( - "Select address,date,type,subject,body FROM sms;"); + "Select address,date,type,subject,body FROM sms;"); //NOI18N BlackboardArtifact bba; String address; // may be phone number, or other addresses @@ -96,11 +96,11 @@ class TextMessageAnalyzer { String subject;//message subject String body; //message body while (resultSet.next()) { - address = resultSet.getString("address"); - date = resultSet.getString("date"); - type = resultSet.getString("type"); - subject = resultSet.getString("subject"); - body = resultSet.getString("body"); + address = resultSet.getString("address"); //NOI18N + date = resultSet.getString("date"); //NOI18N + type = resultSet.getString("type"); //NOI18N + subject = resultSet.getString("subject"); //NOI18N + body = resultSet.getString("body"); //NOI18N bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create Message artifact and then add attributes from result set. @@ -122,18 +122,18 @@ class TextMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NOI18N } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); + logger.log(Level.SEVERE, "Error closing database", e); //NOI18N } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NOI18N } } From a80209836a6d44078ada0314b45f85ff296a9bae Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Tue, 2 Dec 2014 21:43:34 -0800 Subject: [PATCH 13/84] Translation completed --- Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties index e69de29bb2..d54e28a55a 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties @@ -0,0 +1,2 @@ +iOSModuleFactory.moduleDescription=\u30B7\u30B9\u30C6\u30E0\u304A\u3088\u3073\u7B2C\u4E09\u8005\u30A2\u30D7\u30EA\u30C7\u30FC\u30BF\u3092\u62BD\u51FA +iOSModuleFactory.moduleName=iOS\u30A2\u30CA\u30E9\u30A4\u30B6\u30FC \ No newline at end of file From cffe45afefe92684e24b1238f6197bff536215bd Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Tue, 2 Dec 2014 21:56:59 -0800 Subject: [PATCH 14/84] Translation started --- .../autopsy/modules/interestingitems/Bundle_ja.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index e69de29bb2..8486c0f638 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -0,0 +1,4 @@ +FilesIdentifierIngestJobSettingsPanel.border.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u4E2D\u306B\u6709\u52B9\u306B\u3059\u308B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u3092\u9078\u629E\uFF1A +FilesSetPanel.descPanel.border.title=\u6982\u8981 +FilesSetPanel.descriptionPanel.border.title=\u6982\u8981 +FilesSetPanel.ignoreKnownFilesCheckbox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\u3092\u7121\u8996 \ No newline at end of file From 606e8a901da44dcb3364de50961b669cf9049c04 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Wed, 3 Dec 2014 10:58:18 -0500 Subject: [PATCH 15/84] Marked internal/log/exception strings with NON-NLS. Fixed incorrect name of bundle string. Created Bundle_ja.properties. --- .../autopsy/modules/photoreccarver/Bundle.properties | 3 ++- .../modules/photoreccarver/Bundle_ja.properties | 0 .../photoreccarver/PhotoRecCarverFileIngestModule.java | 10 +++++----- 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties index 81d6260109..b9a9d6c609 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties @@ -2,9 +2,10 @@ OpenIDE-Module-Name=PhotoRec Carver Ingest Module OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=PhotoRec Carver ingest module. \n\n Carves unallocated space and feeds the resulting carved files back into the system for processing. OpenIDE-Module-Short-Description=Carves unallocated space and feeds carved files back into the system for processing. -unallocatedSpaceProcessingSettingsError.message="Process Unallocated Space" is not checked. This module is designed to carve unallocated space. Either allow processing of unallocated space, or do not use this module. moduleDisplayName.text=PhotoRec Carver moduleDescription.text=Runs PhotoRec carver against unallocated space on the system. + +unallocatedSpaceProcessingSettingsError.message="Process Unallocated Space" is not checked. This module is designed to carve unallocated space. Either allow processing of unallocated space, or do not use this module. unsupportedOS.message=Module is not supported for other than Windows platforms missingExecutable.message=Unable to locate unallocated carver executable. cannotRunExecutable.message=Unable to execute unallocated carver diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index ffb5447804..7accfe711b 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -113,7 +113,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { PhotoRecCarverFileIngestModule.pathsByJob.put(this.context.getJobId(), new WorkingPaths(outputDirPath, tempDirPath)); } catch (SecurityException | IOException | UnsupportedOperationException ex) { - throw new IngestModule.IngestModuleException(NbBundle.getMessage(this.getClass(), "Utilities.cannotCreateOutputDir.message", ex.getLocalizedMessage())); + throw new IngestModule.IngestModuleException(NbBundle.getMessage(this.getClass(), "cannotCreateOutputDir.message", ex.getLocalizedMessage())); } } } @@ -145,7 +145,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { // Check that we have roughly enough disk space left to complete the operation long freeDiskSpace = IngestServices.getInstance().getFreeDiskSpace(); if ((file.getSize() * 2) > freeDiskSpace) { - logger.log(Level.SEVERE, "PhotoRec error processing {0} with {1} Not enough space on primary disk to carve unallocated space.", + logger.log(Level.SEVERE, "PhotoRec error processing {0} with {1} Not enough space on primary disk to carve unallocated space.", // NON-NLS new Object[]{file.getName(), PhotoRecCarverIngestModuleFactory.getModuleName()}); // NON-NLS return IngestModule.ProcessResult.ERROR; } @@ -163,9 +163,9 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { // Scan the file with Unallocated Carver. ProcessBuilder processAndSettings = new ProcessBuilder( "\"" + executableFile + "\"", - "/d", + "/d", // NON-NLS "\"" + outputDirPath.toAbsolutePath() + File.separator + PHOTOREC_RESULTS_BASE + "\"", - "/cmd", + "/cmd", // NON-NLS "\"" + tempFilePath.toFile() + "\"", "search"); // NON_NLS @@ -194,7 +194,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { if (null != tempFilePath && Files.exists(tempFilePath)) { tempFilePath.toFile().delete(); } - logger.log(Level.SEVERE, "PhotoRec carver returned error exit value = {0} when scanning {1}", + logger.log(Level.SEVERE, "PhotoRec carver returned error exit value = {0} when scanning {1}", // NON-NLS new Object[]{exitValue, file.getName()}); // NON-NLS return IngestModule.ProcessResult.ERROR; } From ce0aa56973da572a6073c53626bbc1a650234901 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 3 Dec 2014 13:19:06 -0500 Subject: [PATCH 16/84] Fix error in image file name referenced in timeline code --- .../org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java b/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java index 5555b0ce2c..68851e9caf 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/events/type/MiscTypes.java @@ -62,7 +62,7 @@ public enum MiscTypes implements EventType, ArtifactEventType { final BlackboardAttribute longEnd = attrMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE_END); return String.format("from %1$g %2$g to %3$g %4$g", latStart.getValueDouble(), longStart.getValueDouble(), latEnd.getValueDouble(), longEnd.getValueDouble()); }), - GPS_TRACKPOINT("Location History", "gps_trackpoint.png", + GPS_TRACKPOINT("Location History", "gps-trackpoint.png", BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, new AttributeExtractor(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME), From 9743174cd427dfb8e463c76a7d4d749beadd7d64 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 12:00:58 -0800 Subject: [PATCH 17/84] Translation completed --- .../interestingitems/Bundle_ja.properties | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index 8486c0f638..9f169c3aa8 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -1,4 +1,55 @@ -FilesIdentifierIngestJobSettingsPanel.border.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u4E2D\u306B\u6709\u52B9\u306B\u3059\u308B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u3092\u9078\u629E\uFF1A +FilesIdentifierIngestJobSettingsPanel.border.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u4E2D\u6709\u52B9\u306B\u3059\u308B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u3092\u9078\u629E\uFF1A FilesSetPanel.descPanel.border.title=\u6982\u8981 FilesSetPanel.descriptionPanel.border.title=\u6982\u8981 -FilesSetPanel.ignoreKnownFilesCheckbox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\u3092\u7121\u8996 \ No newline at end of file +FilesSetPanel.ignoreKnownFilesCheckbox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\u3092\u7121\u8996 +FilesSetPanel.messages.filesSetsMustBeNamed=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u306F\u540D\u524D\u304C\u5FC5\u8981\u3067\u3059\u3002 +FilesSetPanel.nameLabel.text=\u30BB\u30C3\u30C8\u540D\uFF1A +FilesSetPanel.title=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8 +FilesSetRulePanel.dirsRadioButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +FilesSetRulePanel.extensionRadioButton.text=\u62E1\u5F35\u5B50\u306E\u307F +FilesSetRulePanel.filesAndDirsRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u304A\u3088\u3073\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +FilesSetRulePanel.filesRadioButton.text=\u30D5\u30A1\u30A4\u30EB +FilesSetRulePanel.fullNameRadioButton.text=\u30D5\u30EB\u30CD\u30FC\u30E0 +FilesSetRulePanel.jLabel1.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\uFF1A +FilesSetRulePanel.jLabel2.text=\u30CD\u30FC\u30E0\u30D1\u30BF\u30FC\u30F3\uFF1A +FilesSetRulePanel.jLabel3.text=\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\uFF1A +FilesSetRulePanel.messages.emptyNameFilter=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u30EB\u30FC\u30EB\u306F\u691C\u7D22\u3059\u308B\u540D\u524D\u304C\u5FC5\u8981\u3067\u3059\u3002 +FilesSetRulePanel.messages.filesSetRulesMustBeNamed=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u30EB\u30FC\u30EB\u306F\u540D\u524D\u304C\u5FC5\u8981\u3067\u3059\u3002 +FilesSetRulePanel.messages.invalidCharInName=\u6B63\u898F\u8868\u73FE\u4EE5\u5916\u306F\\\u3001/\u3001\:\u3001*\u3001?\u3001"\u3001<\u3001>\u3092\u540D\u524D\u306B\u542B\u3081\u307E\u305B\u3093\u3002 +FilesSetRulePanel.messages.invalidCharInPath=\u6B63\u898F\u8868\u73FE\u4EE5\u5916\u306F\\\u3001\:\u3001*\u3001?\u3001"\u3001<\u3001>\u3092\u30D1\u30B9\u306B\u542B\u3081\u307E\u305B\u3093\u3002 +FilesSetRulePanel.messages.invalidNameRegex=\u6B63\u898F\u8868\u73FE\u306F\u6709\u52B9\u306A\u540D\u524D\u3067\u306F\u3042\u308A\u307E\u305B\u3093\uFF1A\n\n{0} +FilesSetRulePanel.messages.invalidPathRegex=\u6B63\u898F\u8868\u73FE\u306F\u6709\u52B9\u306A\u30D1\u30B9\u3067\u306F\u3042\u308A\u307E\u305B\u3093\uFF1A\n\n{0} +FilesSetRulePanel.nameRegexCheckbox.text=\u6B63\u898F\u8868\u73FE +FilesSetRulePanel.pathRegexCheckBox.text=\u6B63\u898F\u8868\u73FE +FilesSetRulePanel.pathSeparatorInfoLabel.text=/\u3092\u30D1\u30B9\u533A\u5207\u308A\u6587\u5B57\u3068\u3057\u3066\u5229\u7528 +FilesSetRulePanel.ruleNameLabel.text=\u30EB\u30FC\u30EB\u540D\uFF1A +FilesSetRulePanel.title=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u30EB\u30FC\u30EB +InterestingItemDefsPanel.bothRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u304A\u3088\u3073\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +InterestingItemDefsPanel.deleteRuleButton.text=\u30EB\u30FC\u30EB\u524A\u9664 +InterestingItemDefsPanel.deleteSetButton.text=\u30BB\u30C3\u30C8\u524A\u9664 +InterestingItemDefsPanel.dirsRadioButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA +InterestingItemDefsPanel.editRuleButton.text=\u30EB\u30FC\u30EB\u7DE8\u96C6 +InterestingItemDefsPanel.editSetButton.text=\u30BB\u30C3\u30C8\u7DE8\u96C6 +InterestingItemDefsPanel.fileNameExtensionRadioButton.text=\u62E1\u5F35\u5B50\u306E\u307F +InterestingItemDefsPanel.fileNameRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u540D +InterestingItemDefsPanel.fileNameRegexCheckbox.text=\u6B63\u898F\u8868\u73FE +InterestingItemDefsPanel.filesRadioButton.text=\u30D5\u30A1\u30A4\u30EB +InterestingItemDefsPanel.ignoreKnownFilesCheckbox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\u3092\u7121\u8996 +InterestingItemDefsPanel.jLabel1.text=\u30EB\u30FC\u30EB\u8A73\u7D30 +InterestingItemDefsPanel.jLabel2.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\uFF1A +InterestingItemDefsPanel.jLabel3.text=\u30CD\u30FC\u30E0\u30D1\u30BF\u30FC\u30F3 +InterestingItemDefsPanel.jLabel4.text=\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\uFF1A +InterestingItemDefsPanel.jLabel5.text=\u6982\u8981\uFF1A +InterestingItemDefsPanel.jLabel6.text=\u30BB\u30C3\u30C8\u8A73\u7D30 +InterestingItemDefsPanel.jTextArea1.text=\u6307\u5B9A\u3055\u308C\u305F\u6761\u4EF6\u3068\u4E00\u81F4\u3059\u308B\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u7D22\u3059\u308B\u306E\u304C\u53EF\u80FD\u306A\u30E2\u30B8\u30E5\u30FC\u30EB\u3067\u3059\u3002\u5404\u30BB\u30C3\u30C8\u306B\u306F\u30D5\u30A1\u30A4\u30EB\u540D\u304A\u3088\u3073\u30DA\u30A2\u30EC\u30F3\u30C8\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\u3092\u3082\u3068\u306B\u4E00\u81F4\u3059\u308B\u30EB\u30FC\u30EB\u30EA\u30B9\u30C8\u304C\u3042\u308A\u307E\u3059\u3002 +InterestingItemDefsPanel.newRuleButton.text=\u65B0\u898F\u30EB\u30FC\u30EB +InterestingItemDefsPanel.newSetButton.text=\u65B0\u898F\u30BB\u30C3\u30C8 +InterestingItemDefsPanel.rulePathFilterRegexCheckBox.text=\u6B63\u898F\u8868\u73FE +InterestingItemDefsPanel.rulesListLabel.text=\u30EB\u30FC\u30EB\uFF1A +InterestingItemDefsPanel.setsListLabel.text=\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8 +InterestingItemsIdentifierIngestModule.moduleDescription=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u3092\u898B\u3064\u3051\u307E\u3059\u3002 +InterestingItemsIdentifierIngestModule.moduleName=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 +OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OpenIDE-Module-Long-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u3092\u898B\u3064\u3051\u307E\u3059 +OpenIDE-Module-Name=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 +OpenIDE-Module-Short-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB \ No newline at end of file From 03c429fdc3538da0784a244585cffb9756c1a0b8 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Wed, 3 Dec 2014 16:29:37 -0500 Subject: [PATCH 18/84] Marked some internal/log/exception strings with NON-NLS. Extracted UI strings. Created Bundle_ja.properties. Added comment about always true test displaying "Skipped" label. --- .../autopsy/diagnostics/Bundle.properties | 20 +++++- .../autopsy/diagnostics/Bundle_ja.properties | 0 .../autopsy/diagnostics/PerformancePanel.java | 69 ++++++++++++------- .../diagnostics/PerformancePanelAction.java | 1 - 4 files changed, 63 insertions(+), 27 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties index eae8c24793..90ad7a3ffe 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle.properties @@ -1,3 +1,4 @@ +CTL_PerformancePanelAction=Performance Diagnostics PerformancePanel.jLabel1.text=Image Reading: PerformancePanel.jLabel2.text=Database Reading: PerformancePanel.jLabel3.text=CPU: @@ -8,4 +9,21 @@ PerformancePanel.startButton.text=Start PerformancePanel.statusLabel.text=\ PerformancePanel.jLabel4.text=File Reading: PerformancePanel.fileReadLabel.text=\ -PerformancePanel.jLabel5.text=This panel performs a series of tests to help identify bottlenecks in the system. +PerformancePanel.jLabel5.text=This panel performs a series of tests to help identify bottlenecks in the system. +PerformancePanel.title=Performance Diagnostics +PerformancePanel.cpuTest.basemsg=Running CPU Test +PerformancePanel.cpuTest.cpuLabel.md5AlgNotFound.text=MD5 Algorithm not found +PerformancePanel.cpuTest.cpuLabel.MBHashedPerSec.text={0} MB hashed / sec +PerformancePanel.imgTest.statusMsg.runningImgReadTest.text=Running Image Reading Test +PerformancePanel.label.caseNotOpen.text=Case Not Open +PerformancePanel.label.noImgInCase.text=No Images In Case +PerformancePanel.ImgTest.imgLabel.MBReadPerSec.text={0} MB read / sec ({1}) +PerformancePanel.FileReadTest.fileReadLabel.skipped.text=Skipped +PerformancePanel.FileReadTest.statusMsg.runningFileReadTest.text=Running File Reading Test +PerformancePanel.fileReadLabel.imgPathNotExist.text=Image Path Doesn't Exist +PerformancePanel.fileReadLabel.errMakeFileReader.text=Error making file reader +PerformancePanel.ImgTest.fileReadLabel.MBReadPerSec.text={0} MB read / sec ({1}) +PerformancePanel.dbTest.status.running=Running DB Test +PerformancePanel.dbTest.dbLabel.recordsPerSec.text={0} records / sec +PerformancePanel.dbTest.dbLabel.errPerformQuery.text=Error Performing Query +PerformancePanel.done.statusMsg.err.text=Error\: {0} diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java index e2cc873095..ace829ee2c 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanel.java @@ -34,6 +34,8 @@ import java.util.concurrent.ExecutionException; import javax.swing.JFrame; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; + +import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.datamodel.AbstractFile; @@ -48,7 +50,8 @@ public class PerformancePanel extends javax.swing.JDialog { * Creates new form PerformancePanel */ public PerformancePanel() { - super((JFrame) WindowManager.getDefault().getMainWindow(), "Performance Diagnostics", true); + super((JFrame) WindowManager.getDefault().getMainWindow(), + NbBundle.getMessage(PerformancePanel.class, "PerformancePanel.title"), true); initComponents(); } @@ -258,14 +261,15 @@ public class PerformancePanel extends javax.swing.JDialog { private void doCpuTest() { - final String msg = "Running CPU Test"; + final String msg = NbBundle.getMessage(this.getClass(), "PerformancePanel.cpuTest.basemsg"); MessageDigest md; long start = new Date().getTime(); try { - md = MessageDigest.getInstance("MD5"); + md = MessageDigest.getInstance("MD5"); // NON-NLS } catch (NoSuchAlgorithmException ex) { - setCpuLabel("MD5 Algo not found"); + setCpuLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.cpuTest.cpuLabel.md5AlgNotFound.text")); return; } @@ -285,20 +289,22 @@ public class PerformancePanel extends javax.swing.JDialog { long end = new Date().getTime(); cpuStats = (bytesRead / (1024 * 1024)) / ((end - start) / 1000); - setCpuLabel(cpuStats + " MB hashed / sec"); + setCpuLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.cpuTest.cpuLabel.MBHashedPerSec.text", + cpuStats)); setStatusMsg(""); } private void doImgTest() { imgReadStats = 0; - setStatusMsg("Running Image Reading Test"); + setStatusMsg( + NbBundle.getMessage(this.getClass(), "PerformancePanel.imgTest.statusMsg.runningImgReadTest.text")); Case curCase; try { curCase = Case.getCurrentCase(); } catch (Exception e) { - setImgLabel("Case Not Open"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.caseNotOpen.text")); setStatusMsg(""); return; } @@ -307,7 +313,7 @@ public class PerformancePanel extends javax.swing.JDialog { try { dataSources = curCase.getDataSources(); } catch (TskCoreException ex) { - setImgLabel("No Images In Case"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } @@ -318,7 +324,7 @@ public class PerformancePanel extends javax.swing.JDialog { } } if (image == null) { - setImgLabel("No Images In Case"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } @@ -356,25 +362,30 @@ public class PerformancePanel extends javax.swing.JDialog { imgReadStats = (bytesRead / (1024 * 1024)) / elapsed; else imgReadStats = 0; - setImgLabel(imgReadStats + " MB read / sec (" + bytesRead + ")"); + setImgLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.ImgTest.imgLabel.MBReadPerSec.text", + imgReadStats, bytesRead)); setStatusMsg(""); } private void doFileReadTest() { fileReadStats = 0; - + + // TODO: this is always true. Why display a "Skipped" label and then go on to run the test? if (true) { - setFileReadLabel("Skipped"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.FileReadTest.fileReadLabel.skipped.text")); } - setStatusMsg("Running File Reading Test"); + setStatusMsg(NbBundle.getMessage(this.getClass(), + "PerformancePanel.FileReadTest.statusMsg.runningFileReadTest.text")); Case curCase; try { curCase = Case.getCurrentCase(); } catch (Exception e) { - setFileReadLabel("Case Not Open"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.label.caseNotOpen.text")); setStatusMsg(""); return; } @@ -383,7 +394,8 @@ public class PerformancePanel extends javax.swing.JDialog { try { dataSources = curCase.getDataSources(); } catch (TskCoreException ex) { - setFileReadLabel("No Images In Case"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } @@ -394,14 +406,16 @@ public class PerformancePanel extends javax.swing.JDialog { } } if (image == null) { - setFileReadLabel("No Images In Case"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.label.noImgInCase.text")); setStatusMsg(""); return; } File file = new File(image.getPaths()[0]); if (file.exists() == false) { - setFileReadLabel("Image Path Doesn't Exist"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.fileReadLabel.imgPathNotExist.text")); setStatusMsg(""); return; } @@ -410,7 +424,8 @@ public class PerformancePanel extends javax.swing.JDialog { try { fileReader = new FileReader(file); } catch (FileNotFoundException ex) { - setFileReadLabel("Error making file reader"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.fileReadLabel.errMakeFileReader.text")); setStatusMsg(""); return; } @@ -451,20 +466,22 @@ public class PerformancePanel extends javax.swing.JDialog { fileReadStats = (bytesRead / (1024 * 1024)) / elapsed; else fileReadStats = 0; - setFileReadLabel(fileReadStats + " MB read / sec (" + bytesRead + ")"); + setFileReadLabel( + NbBundle.getMessage(this.getClass(), "PerformancePanel.ImgTest.fileReadLabel.MBReadPerSec.text", + fileReadStats, bytesRead)); setStatusMsg(""); } private void doDbTest() { dbStats = 0; - setStatusMsg("Running DB Test"); + setStatusMsg(NbBundle.getMessage(this.getClass(), "PerformancePanel.dbTest.status.running")); Case curCase; try { curCase = Case.getCurrentCase(); } catch (Exception e) { - setDbLabel("Case Not Open"); + setDbLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.label.caseNotOpen.text")); return; } @@ -472,7 +489,7 @@ public class PerformancePanel extends javax.swing.JDialog { SleuthkitCase tskCase = curCase.getSleuthkitCase(); long start = new Date().getTime(); - List files = tskCase.findAllFilesWhere("obj_id < 50000"); + List files = tskCase.findAllFilesWhere("obj_id < 50000"); // NON-NLS long end = new Date().getTime(); long elapsed = (end - start) / 1000; @@ -481,9 +498,10 @@ public class PerformancePanel extends javax.swing.JDialog { else dbStats = 0; - setDbLabel(dbStats + " records / sec"); + setDbLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.dbTest.dbLabel.recordsPerSec.text", + dbStats)); } catch (TskCoreException ex) { - setDbLabel("Error Performing Query"); + setDbLabel(NbBundle.getMessage(this.getClass(), "PerformancePanel.dbTest.dbLabel.errPerformQuery.text")); } setStatusMsg(""); @@ -494,7 +512,8 @@ public class PerformancePanel extends javax.swing.JDialog { try { get(); } catch (InterruptedException | ExecutionException ex) { - setStatusMsg("Error: " + ex.getMessage()); + setStatusMsg(NbBundle.getMessage(this.getClass(), "PerformancePanel.done.statusMsg.err.text", + ex.getMessage())); } startButton.setEnabled(true); } diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java index 0f962e6abf..54559d0f9c 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java @@ -35,7 +35,6 @@ import org.openide.util.NbBundle.Messages; displayName = "#CTL_PerformancePanelAction" ) @ActionReference(path = "Menu/Help", position = 1437) -@Messages("CTL_PerformancePanelAction=Performance Diagnostics") public final class PerformancePanelAction implements ActionListener { @Override From fd73aff88b8bdf3177a45ce5bbf82916f4e39587 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 13:51:00 -0800 Subject: [PATCH 19/84] Translation completed --- Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties index d54e28a55a..38b86e4f10 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/Bundle_ja.properties @@ -1,2 +1,2 @@ iOSModuleFactory.moduleDescription=\u30B7\u30B9\u30C6\u30E0\u304A\u3088\u3073\u7B2C\u4E09\u8005\u30A2\u30D7\u30EA\u30C7\u30FC\u30BF\u3092\u62BD\u51FA -iOSModuleFactory.moduleName=iOS\u30A2\u30CA\u30E9\u30A4\u30B6\u30FC \ No newline at end of file +iOSModuleFactory.moduleName=iOS\u30A2\u30CA\u30E9\u30A4\u30B6 \ No newline at end of file From 1a93da00995591d00840b25ee9a1627784d08615 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 13:55:07 -0800 Subject: [PATCH 20/84] Translation completed --- .../org/sleuthkit/autopsy/modules/android/Bundle_ja.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties index e69de29bb2..dc3ed0fa78 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties @@ -0,0 +1,2 @@ +AndroidModuleFactory.moduleName= +AndroidModuleFactory.moduleDescription=Android\u30B7\u30B9\u30C6\u30E0\u304A\u3088\u3073\u7B2C\u4E09\u8005\u30A2\u30D7\u30EA\u30C7\u30FC\u30BF\u3092\u62BD\u51FA \ No newline at end of file From c2a36a8c95826fbb84b87a168c8615c3f3983509 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 16:42:28 -0800 Subject: [PATCH 21/84] Translation complete --- .../autopsy/modules/photoreccarver/Bundle_ja.properties | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties index e69de29bb2..ed5e42a0a5 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties @@ -0,0 +1,6 @@ +moduleDescription.text=\u30B7\u30B9\u30C6\u30E0\u306E\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u306B\u5BFE\u3057\u3066PhotoRec\u30AB\u30FC\u30D0\u3092\u5B9F\u884C\u3057\u307E\u3059\u3002 +moduleDisplayName.text=PhotoRec\u30AB\u30FC\u30D0 +OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OpenIDE-Module-Long-Description=PhotoRec\u30AB\u30FC\u30D0\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u5207\u308A\u51FA\u3057\u3001\u51E6\u7406\u3059\u308B\u3081\u306B\u30B7\u30B9\u30C6\u30E0\u3078\u30D5\u30A3\u30FC\u30C9\u3057\u307E\u3059\u3002 +OpenIDE-Module-Name=PhotoRec\u30AB\u30FC\u30D0\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OpenIDE-Module-Short-Description=\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u5207\u308A\u51FA\u3057\u3001\u51E6\u7406\u3059\u308B\u3081\u306B\u30B7\u30B9\u30C6\u30E0\u3078\u30D5\u30A3\u30FC\u30C9\u3057\u307E\u3059\u3002 \ No newline at end of file From e723a1f365e060077144169ea7683376efcb901d Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 18:12:05 -0800 Subject: [PATCH 22/84] Deleted empty string --- .../org/sleuthkit/autopsy/modules/android/Bundle_ja.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties index dc3ed0fa78..a091c264ff 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/android/Bundle_ja.properties @@ -1,2 +1 @@ -AndroidModuleFactory.moduleName= AndroidModuleFactory.moduleDescription=Android\u30B7\u30B9\u30C6\u30E0\u304A\u3088\u3073\u7B2C\u4E09\u8005\u30A2\u30D7\u30EA\u30C7\u30FC\u30BF\u3092\u62BD\u51FA \ No newline at end of file From 42bf51b66fc40d7651a59be641ac6622473636fc Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 20:13:27 -0800 Subject: [PATCH 23/84] Translation started --- .../autopsy/diagnostics/Bundle_ja.properties | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties index e69de29bb2..09ec101d1d 100644 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/Bundle_ja.properties @@ -0,0 +1,22 @@ +CTL_PerformancePanelAction=\u30D1\u30D5\u30A9\u30FC\u30DE\u30F3\u30B9\u8A3A\u65AD +PerformancePanel.cpuTest.basemsg=CPU\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.cpuTest.cpuLabel.MBHashedPerSec.text=\uFF11\u79D2\u3054\u3068\u306B{0}MB\u30CF\u30C3\u30B7\u30E5\u5B8C\u4E86 +PerformancePanel.cpuTest.cpuLabel.md5AlgNotFound.text=MD5\u30A2\u30EB\u30B4\u30EA\u30BA\u30E0\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F +PerformancePanel.dbTest.dbLabel.errPerformQuery.text=\u30AF\u30A8\u30EA\u5B9F\u884C\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F +PerformancePanel.dbTest.dbLabel.recordsPerSec.text=\uFF11\u79D2\u3054\u3068\u306B{0}\u30EC\u30B3\u30FC\u30C9 +PerformancePanel.dbTest.status.running=DB\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.done.statusMsg.err.text=\u30A8\u30E9\u30FC\uFF1A{0} +PerformancePanel.fileReadLabel.errMakeFileReader.text=\u30D5\u30A1\u30A4\u30EB\u30EA\u30FC\u30C0\u30FC\u4F5C\u6210\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F +PerformancePanel.fileReadLabel.imgPathNotExist.text=\u30A4\u30E1\u30FC\u30B8\u30D1\u30B9\u304C\u5B58\u5728\u3057\u307E\u305B\u3093 +PerformancePanel.FileReadTest.fileReadLabel.skipped.text=\u30B9\u30AD\u30C3\u30D7\u3055\u308C\u307E\u3057\u305F +PerformancePanel.FileReadTest.statusMsg.runningFileReadTest.text=\u30D5\u30A1\u30A4\u30EB\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.imgTest.statusMsg.runningImgReadTest.text=\u30A4\u30E1\u30FC\u30B8\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\u30C6\u30B9\u30C8\u5B9F\u884C\u4E2D +PerformancePanel.jLabel1.text=\u30A4\u30E1\u30FC\u30B8\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\uFF1A +PerformancePanel.jLabel2.text=\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\uFF1A +PerformancePanel.jLabel3.text=CPU\uFF1A +PerformancePanel.jLabel4.text=\u30D5\u30A1\u30A4\u30EB\u30EA\u30FC\u30C7\u30A3\u30F3\u30B0\uFF1A +PerformancePanel.jLabel5.text=\u30B7\u30B9\u30C6\u30E0\u306E\u30DC\u30C8\u30EB\u30CD\u30C3\u30AF\u3092\u7279\u5B9A\u3059\u308B\u4E00\u9023\u306E\u30C6\u30B9\u30C8\u3092\u884C\u3046\u30D1\u30CD\u30EB\u3067\u3059\u3002 +PerformancePanel.label.caseNotOpen.text=\u30B1\u30FC\u30B9\u304C\u958B\u3044\u3066\u3044\u307E\u305B\u3093 +PerformancePanel.label.noImgInCase.text=\u30B1\u30FC\u30B9\u306B\u30A4\u30E1\u30FC\u30B8\u304C\u3042\u308A\u307E\u305B\u3093 +PerformancePanel.startButton.text=\u958B\u59CB +PerformancePanel.title=\u30D1\u30D5\u30A9\u30FC\u30DE\u30F3\u30B9\u8A3A\u65AD \ No newline at end of file From 7c989a1fff95ab355ab58f7e707a6457525bb246 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 20:21:15 -0800 Subject: [PATCH 24/84] Made minor changes to make it more consistent. Translation completed. --- .../autopsy/modules/interestingitems/Bundle_ja.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index 9f169c3aa8..30b5208be7 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -25,11 +25,11 @@ FilesSetRulePanel.pathSeparatorInfoLabel.text=/\u3092\u30D1\u30B9\u533A\u5207\u3 FilesSetRulePanel.ruleNameLabel.text=\u30EB\u30FC\u30EB\u540D\uFF1A FilesSetRulePanel.title=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30BB\u30C3\u30C8\u30EB\u30FC\u30EB InterestingItemDefsPanel.bothRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u304A\u3088\u3073\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA -InterestingItemDefsPanel.deleteRuleButton.text=\u30EB\u30FC\u30EB\u524A\u9664 -InterestingItemDefsPanel.deleteSetButton.text=\u30BB\u30C3\u30C8\u524A\u9664 +InterestingItemDefsPanel.deleteRuleButton.text=\u30EB\u30FC\u30EB\u3092\u524A\u9664 +InterestingItemDefsPanel.deleteSetButton.text=\u30BB\u30C3\u30C8\u3092\u524A\u9664 InterestingItemDefsPanel.dirsRadioButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA -InterestingItemDefsPanel.editRuleButton.text=\u30EB\u30FC\u30EB\u7DE8\u96C6 -InterestingItemDefsPanel.editSetButton.text=\u30BB\u30C3\u30C8\u7DE8\u96C6 +InterestingItemDefsPanel.editRuleButton.text=\u30EB\u30FC\u30EB\u3092\u7DE8\u96C6 +InterestingItemDefsPanel.editSetButton.text=\u30BB\u30C3\u30C8\u3092\u7DE8\u96C6 InterestingItemDefsPanel.fileNameExtensionRadioButton.text=\u62E1\u5F35\u5B50\u306E\u307F InterestingItemDefsPanel.fileNameRadioButton.text=\u30D5\u30A1\u30A4\u30EB\u540D InterestingItemDefsPanel.fileNameRegexCheckbox.text=\u6B63\u898F\u8868\u73FE From 0e00d6cfed86f870021e17491cdad770c680f4c9 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Wed, 3 Dec 2014 20:23:40 -0800 Subject: [PATCH 25/84] Made minor changes for better translation. Translation completed. --- .../autopsy/modules/photoreccarver/Bundle_ja.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties index ed5e42a0a5..45cdd48789 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle_ja.properties @@ -3,4 +3,4 @@ moduleDisplayName.text=PhotoRec\u30AB\u30FC\u30D0 OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB OpenIDE-Module-Long-Description=PhotoRec\u30AB\u30FC\u30D0\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u5207\u308A\u51FA\u3057\u3001\u51E6\u7406\u3059\u308B\u3081\u306B\u30B7\u30B9\u30C6\u30E0\u3078\u30D5\u30A3\u30FC\u30C9\u3057\u307E\u3059\u3002 OpenIDE-Module-Name=PhotoRec\u30AB\u30FC\u30D0\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB -OpenIDE-Module-Short-Description=\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u5207\u308A\u51FA\u3057\u3001\u51E6\u7406\u3059\u308B\u3081\u306B\u30B7\u30B9\u30C6\u30E0\u3078\u30D5\u30A3\u30FC\u30C9\u3057\u307E\u3059\u3002 \ No newline at end of file +OpenIDE-Module-Short-Description=\u51E6\u7406\u3059\u308B\u3081\u306B\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u5207\u308A\u51FA\u3057\u3001\u30B7\u30B9\u30C6\u30E0\u3078\u30D5\u30A3\u30FC\u30C9\u3057\u307E\u3059\u3002 \ No newline at end of file From ad9cb8228d723b4effa2cdd9a85a82d8e62ec5c1 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Thu, 4 Dec 2014 12:45:24 -0500 Subject: [PATCH 26/84] Extracted one string into Bundle. --- Core/src/org/sleuthkit/autopsy/actions/Bundle.properties | 1 + .../sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties index 3636fce22f..6c3e5ce690 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties @@ -52,3 +52,4 @@ CTL_OpenLogFolder=Open Log Folder ShowIngestProgressSnapshotAction.actionName.text=Get Ingest Progress Snapshot OpenPythonModulesFolderAction.actionName.text=Python Plugins OpenPythonModulesFolderAction.errorMsg.folderNotFound=Python plugins folder not found: {0} +CTL_OpenPythonModulesFolderAction=Python Plugins \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java b/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java index 68195c299a..16a3ea1891 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/OpenPythonModulesFolderAction.java @@ -38,7 +38,6 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; @ActionID(category = "Tools", id = "org.sleuthkit.autopsy.actions.OpenPythonModulesFolderAction") @ActionRegistration(displayName = "#CTL_OpenPythonModulesFolderAction", lazy = false) @ActionReference(path = "Menu/Tools", position = 1400) -@NbBundle.Messages("CTL_OpenPythonModulesFolderAction=Python Plugins") public final class OpenPythonModulesFolderAction extends SystemAction implements ActionListener { private static final Logger logger = Logger.getLogger(OpenPythonModulesFolderAction.class.getName()); From 2d29876f789144ea3587ff50a5e39c70e8517c38 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Thu, 4 Dec 2014 12:58:29 -0500 Subject: [PATCH 27/84] Replaced NOI18N with NON-NLS to be consistent. --- .../modules/android/AndroidIngestModule.java | 26 ++++++------- .../android/BrowserLocationAnalyzer.java | 24 ++++++------ .../android/CacheLocationAnalyzer.java | 8 ++-- .../modules/android/CallLogAnalyzer.java | 32 ++++++++-------- .../modules/android/ContactAnalyzer.java | 38 +++++++++---------- .../android/GoogleMapLocationAnalyzer.java | 32 ++++++++-------- .../modules/android/TangoMessageAnalyzer.java | 32 ++++++++-------- .../modules/android/TextMessageAnalyzer.java | 30 +++++++-------- .../modules/android/WWFMessageAnalyzer.java | 22 +++++------ 9 files changed, 122 insertions(+), 122 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java index ee0dac4221..52439eb6fd 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/AndroidIngestModule.java @@ -57,7 +57,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Contacts"); //NOI18N + errors.add("Error getting Contacts"); //NON-NLS } try { @@ -67,7 +67,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Call Logs"); //NOI18N + errors.add("Error getting Call Logs"); //NON-NLS } try { @@ -77,7 +77,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Text Messages"); //NOI18N + errors.add("Error getting Text Messages"); //NON-NLS } try { @@ -87,7 +87,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Tango Messages"); //NOI18N + errors.add("Error getting Tango Messages"); //NON-NLS } try { @@ -97,7 +97,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Words with Friends Messages"); //NOI18N + errors.add("Error getting Words with Friends Messages"); //NON-NLS } try { @@ -107,7 +107,7 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Google Map Locations"); //NOI18N + errors.add("Error getting Google Map Locations"); //NON-NLS } try { @@ -117,14 +117,14 @@ class AndroidIngestModule implements DataSourceIngestModule { return IngestModule.ProcessResult.OK; } } catch (Exception e) { - errors.add("Error getting Browser Locations"); //NOI18N + errors.add("Error getting Browser Locations"); //NON-NLS } try { CacheLocationAnalyzer.findGeoLocations(); progressBar.progress(8); } catch (Exception e) { - errors.add("Error getting Cache Locations"); //NOI18N + errors.add("Error getting Cache Locations"); //NON-NLS } // create the final message for inbox @@ -133,20 +133,20 @@ class AndroidIngestModule implements DataSourceIngestModule { IngestMessage.MessageType msgLevel = IngestMessage.MessageType.INFO; if (errors.isEmpty() == false) { msgLevel = IngestMessage.MessageType.ERROR; - errorMessage.append("Errors were encountered"); //NOI18N + errorMessage.append("Errors were encountered"); //NON-NLS for (String msg : errors) { errorMessage.append("
  • ").append(msg).append("
    • \n"); //NON-NLS } errorMessage.append("\n"); //NON-NLS if (errors.size() == 1) { - errorMsgSubject = "One error was found"; //NOI18N + errorMsgSubject = "One error was found"; //NON-NLS } else { - errorMsgSubject = "errors found: " + errors.size(); //NOI18N + errorMsgSubject = "errors found: " + errors.size(); //NON-NLS } } else { - errorMessage.append("No errors"); //NOI18N - errorMsgSubject = "No errors"; //NOI18N + errorMessage.append("No errors"); //NON-NLS + errorMsgSubject = "No errors"; //NON-NLS } services.postMessage(IngestMessage.createMessage(msgLevel, AndroidModuleFactory.getModuleName(), "Finished Analysis: " + errorMsgSubject, errorMessage.toString())); diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java index acf6774a6f..339a7d5fca 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/BrowserLocationAnalyzer.java @@ -43,7 +43,7 @@ class BrowserLocationAnalyzer { public static void findGeoLocations() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List abstractFiles = skCase.findAllFilesWhere("name LIKE 'CachedGeoposition%.db'"); //NOI18N //get exact file names + List abstractFiles = skCase.findAllFilesWhere("name LIKE 'CachedGeoposition%.db'"); //NON-NLS //get exact file names for (AbstractFile abstractFile : abstractFiles) { try { @@ -54,11 +54,11 @@ class BrowserLocationAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findGeoLocationsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Browser Location files", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Browser Location files", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Browser Location files", e); //NOI18N + logger.log(Level.SEVERE, "Error finding Browser Location files", e); //NON-NLS } } @@ -71,22 +71,22 @@ class BrowserLocationAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error connecting to sql database", e); //NOI18N + logger.log(Level.SEVERE, "Error connecting to sql database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select timestamp, latitude, longitude, accuracy FROM CachedPosition;"); //NOI18N + "Select timestamp, latitude, longitude, accuracy FROM CachedPosition;"); //NON-NLS while (resultSet.next()) { - Long timestamp = Long.valueOf(resultSet.getString("timestamp")) / 1000; //NOI18N - double latitude = Double.valueOf(resultSet.getString("latitude")); //NOI18N - double longitude = Double.valueOf(resultSet.getString("longitude")); //NOI18N + Long timestamp = Long.valueOf(resultSet.getString("timestamp")) / 1000; //NON-NLS + double latitude = Double.valueOf(resultSet.getString("latitude")); //NON-NLS + double longitude = Double.valueOf(resultSet.getString("longitude")); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), moduleName, latitude)); @@ -96,7 +96,7 @@ class BrowserLocationAnalyzer { // bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VALUE.getTypeID(),moduleName, accuracy)); } } catch (Exception e) { - logger.log(Level.SEVERE, "Error Putting artifacts to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error Putting artifacts to Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -105,7 +105,7 @@ class BrowserLocationAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java index 6e5bae3027..3e942136f8 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/CacheLocationAnalyzer.java @@ -43,7 +43,7 @@ class CacheLocationAnalyzer { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List abstractFiles = skCase.findAllFilesWhere("name ='cache.cell' OR name='cache.wifi'"); //NOI18N //get exact file names + List abstractFiles = skCase.findAllFilesWhere("name ='cache.cell' OR name='cache.wifi'"); //NON-NLS //get exact file names for (AbstractFile abstractFile : abstractFiles) { try { @@ -55,11 +55,11 @@ class CacheLocationAnalyzer { findGeoLocationsInFile(jFile, abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing cached Location files", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing cached Location files", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding cached Location files", e); //NOI18N + logger.log(Level.SEVERE, "Error finding cached Location files", e); //NON-NLS } } @@ -124,7 +124,7 @@ class CacheLocationAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Cached GPS locations to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Cached GPS locations to Blackboard", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java index b213e95004..8c8542f789 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/CallLogAnalyzer.java @@ -47,11 +47,11 @@ class CallLogAnalyzer { /** the where clause(without 'where' of sql select statement to choose call * log dbs, update the list of file names to include more files */ - private static final String fileNameQuery = Stream.of("'logs.db'", "'contacts2.db'", "'contacts.db'") //NOI18N - .collect(Collectors.joining(" OR name = ", "name = ", "")); //NOI18N + private static final String fileNameQuery = Stream.of("'logs.db'", "'contacts2.db'", "'contacts.db'") //NON-NLS + .collect(Collectors.joining(" OR name = ", "name = ", "")); //NON-NLS /** the names of tables that potentially hold call logs in the dbs */ - private static final Iterable tableNames = Arrays.asList("calls", "logs"); //NOI18N + private static final Iterable tableNames = Arrays.asList("calls", "logs"); //NON-NLS public static void findCallLogs() { try { @@ -63,11 +63,11 @@ class CallLogAnalyzer { ContentUtils.writeToFile(abstractFile, file); findCallLogsInDB(file.toString(), abstractFile); } catch (IOException e) { - logger.log(Level.SEVERE, "Error writing temporary call log db to disk", e); //NOI18N + logger.log(Level.SEVERE, "Error writing temporary call log db to disk", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding call logs", e); //NOI18N + logger.log(Level.SEVERE, "Error finding call logs", e); //NON-NLS } } @@ -76,20 +76,20 @@ class CallLogAnalyzer { if (DatabasePath == null || DatabasePath.isEmpty()) { return; } - try (Connection connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + try (Connection connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS Statement statement = connection.createStatement();) { for (String tableName : tableNames) { try (ResultSet resultSet = statement.executeQuery( - "SELECT number,date,duration,type, name FROM " + tableName + " ORDER BY date DESC;");) { //NOI18N - logger.log(Level.INFO, "Reading call log from table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NOI18N + "SELECT number,date,duration,type, name FROM " + tableName + " ORDER BY date DESC;");) { //NON-NLS + logger.log(Level.INFO, "Reading call log from table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NON-NLS while (resultSet.next()) { Long date = resultSet.getLong("date") / 1000; - final CallDirection direction = CallDirection.fromType(resultSet.getInt("type")); //NOI18N + final CallDirection direction = CallDirection.fromType(resultSet.getInt("type")); //NON-NLS String directionString = direction != null ? direction.getDisplayName() : ""; - final String number = resultSet.getString("number"); //NOI18N - final long duration = resultSet.getLong("duration"); //NOI18N //duration of call is in seconds - final String name = resultSet.getString("name"); //NOI18N // name of person dialed or called. null if unregistered + final String number = resultSet.getString("number"); //NON-NLS + final long duration = resultSet.getLong("duration"); //NON-NLS //duration of call is in seconds + final String name = resultSet.getString("name"); //NON-NLS // name of person dialed or called. null if unregistered try { BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); //create a call log and then add attributes from result set. @@ -104,21 +104,21 @@ class CallLogAnalyzer { bba.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION.getTypeID(), moduleName, directionString)); bba.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error posting call log record to the Blackboard", ex); //NOI18N + logger.log(Level.SEVERE, "Error posting call log record to the Blackboard", ex); //NON-NLS } } } catch (SQLException e) { - logger.log(Level.WARNING, "Could not read table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NOI18N + logger.log(Level.WARNING, "Could not read table {0} in db {1}", new Object[]{tableName, DatabasePath}); //NON-NLS } } } catch (SQLException e) { - logger.log(Level.SEVERE, "Could not parse call log; error connecting to db " + DatabasePath, e); //NOI18N + logger.log(Level.SEVERE, "Could not parse call log; error connecting to db " + DatabasePath, e); //NON-NLS } } private static enum CallDirection { - INCOMING(1, "Incoming"), OUTGOING(2, "Outgoing"), MISSED(3, "Missed"); //NOI18N + INCOMING(1, "Incoming"), OUTGOING(2, "Outgoing"), MISSED(3, "Missed"); //NON-NLS private final int type; diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java index 68e021f0fa..3d4784864f 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/ContactAnalyzer.java @@ -45,7 +45,7 @@ class ContactAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NOI18N //get exact file names + absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NON-NLS //get exact file names if (absFiles.isEmpty()) { return; } @@ -55,11 +55,11 @@ class ContactAnalyzer { ContentUtils.writeToFile(AF, jFile); findContactsInDB(jFile.toString(), AF); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Contacts", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Contacts", e); //NOI18N + logger.log(Level.SEVERE, "Error finding Contacts", e); //NON-NLS } } @@ -78,11 +78,11 @@ class ContactAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } @@ -90,13 +90,13 @@ class ContactAnalyzer { // get display_name, mimetype(email or phone number) and data1 (phonenumber or email address depending on mimetype) //sorted by name, so phonenumber/email would be consecutive for a person if they exist. resultSet = statement.executeQuery( - "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NOI18N - + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NOI18N - + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NOI18N - + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NOI18N - + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NOI18N - + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NOI18N - + "ORDER BY name_raw_contact.display_name ASC;"); //NOI18N + "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NON-NLS + + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NON-NLS + + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NON-NLS + + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NON-NLS + + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NON-NLS + + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NON-NLS + + "ORDER BY name_raw_contact.display_name ASC;"); //NON-NLS BlackboardArtifact bba; bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); @@ -105,15 +105,15 @@ class ContactAnalyzer { String mimetype; // either phone or email String data1; // the phone number or email while (resultSet.next()) { - name = resultSet.getString("display_name"); //NOI18N - data1 = resultSet.getString("data1"); //NOI18N - mimetype = resultSet.getString("mimetype"); //NOI18N + name = resultSet.getString("display_name"); //NON-NLS + data1 = resultSet.getString("data1"); //NON-NLS + mimetype = resultSet.getString("mimetype"); //NON-NLS // System.out.println(resultSet.getString("data1") + resultSet.getString("mimetype") + resultSet.getString("display_name")); //Test code if (name.equals(oldName) == false) { bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } - if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NOI18N + if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getTypeID(), moduleName, data1)); } else { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getTypeID(), moduleName, data1)); @@ -122,7 +122,7 @@ class ContactAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -131,7 +131,7 @@ class ContactAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java index 09925e3a8f..88f1aa87ec 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/GoogleMapLocationAnalyzer.java @@ -44,7 +44,7 @@ class GoogleMapLocationAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='da_destination_history'"); //NOI18N //get exact file name + absFiles = skCase.findAllFilesWhere("name ='da_destination_history'"); //NON-NLS //get exact file name if (absFiles.isEmpty()) { return; } @@ -54,11 +54,11 @@ class GoogleMapLocationAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findGeoLocationsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Google map locations", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Google map locations", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Google map locations", e); //NOI18N + logger.log(Level.SEVERE, "Error finding Google map locations", e); //NON-NLS } } @@ -71,27 +71,27 @@ class GoogleMapLocationAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select time,dest_lat,dest_lng,dest_title,dest_address,source_lat,source_lng FROM destination_history;"); //NOI18N + "Select time,dest_lat,dest_lng,dest_title,dest_address,source_lat,source_lng FROM destination_history;"); //NON-NLS while (resultSet.next()) { - Long time = Long.valueOf(resultSet.getString("time")) / 1000; //NOI18N - String dest_title = resultSet.getString("dest_title"); //NOI18N - String dest_address = resultSet.getString("dest_address"); //NOI18N + Long time = Long.valueOf(resultSet.getString("time")) / 1000; //NON-NLS + String dest_title = resultSet.getString("dest_title"); //NON-NLS + String dest_address = resultSet.getString("dest_address"); //NON-NLS - double dest_lat = convertGeo(resultSet.getString("dest_lat")); //NOI18N - double dest_lng = convertGeo(resultSet.getString("dest_lng")); //NOI18N - double source_lat = convertGeo(resultSet.getString("source_lat")); //NOI18N - double source_lng = convertGeo(resultSet.getString("source_lng")); //NOI18N + double dest_lat = convertGeo(resultSet.getString("dest_lat")); //NON-NLS + double dest_lng = convertGeo(resultSet.getString("dest_lng")); //NON-NLS + double source_lat = convertGeo(resultSet.getString("source_lat")); //NON-NLS + double source_lng = convertGeo(resultSet.getString("source_lng")); //NON-NLS // bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT);//src @@ -123,7 +123,7 @@ class GoogleMapLocationAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Google map locations to the Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Google map locations to the Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -132,7 +132,7 @@ class GoogleMapLocationAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing the database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing the database", e); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java index 5b8cee3b4d..cdae0bfcad 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/TangoMessageAnalyzer.java @@ -45,18 +45,18 @@ class TangoMessageAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='tc.db' "); //NOI18N //get exact file names + absFiles = skCase.findAllFilesWhere("name ='tc.db' "); //NON-NLS //get exact file names for (AbstractFile abstractFile : absFiles) { try { File jFile = new File(Case.getCurrentCase().getTempDirectory(), abstractFile.getName()); ContentUtils.writeToFile(abstractFile, jFile); findTangoMessagesInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Tango messages", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Tango messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Tango messages", e); //NOI18N + logger.log(Level.SEVERE, "Error finding Tango messages", e); //NON-NLS } } @@ -69,31 +69,31 @@ class TangoMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select conv_id, create_time,direction,payload FROM messages ORDER BY create_time DESC;"); //NOI18N + "Select conv_id, create_time,direction,payload FROM messages ORDER BY create_time DESC;"); //NON-NLS String conv_id; // seems to wrap around the message found in payload after decoding from base-64 String direction; // 1 incoming, 2 outgoing String payload; // seems to be a base64 message wrapped by the conv_id while (resultSet.next()) { - conv_id = resultSet.getString("conv_id"); //NOI18N - Long create_time = Long.valueOf(resultSet.getString("create_time")) / 1000; //NOI18N - if (resultSet.getString("direction").equals("1")) { //NOI18N - direction = "Incoming"; //NOI18N + conv_id = resultSet.getString("conv_id"); //NON-NLS + Long create_time = Long.valueOf(resultSet.getString("create_time")) / 1000; //NON-NLS + if (resultSet.getString("direction").equals("1")) { //NON-NLS + direction = "Incoming"; //NON-NLS } else { - direction = "Outgoing"; //NOI18N + direction = "Outgoing"; //NON-NLS } - payload = resultSet.getString("payload"); //NOI18N + payload = resultSet.getString("payload"); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create a call log and then add attributes from result set. bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), moduleName, create_time)); @@ -104,7 +104,7 @@ class TangoMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Tango messages to the Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Tango messages to the Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -113,7 +113,7 @@ class TangoMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } @@ -126,7 +126,7 @@ class TangoMessageAnalyzer { String Z = new String(decoded, "UTF-8"); result = Z.split(wrapper)[1]; } catch (Exception e) { - logger.log(Level.SEVERE, "Error decoding a Tango message", e); //NOI18N + logger.log(Level.SEVERE, "Error decoding a Tango message", e); //NON-NLS } return result; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java index 0b58b75313..10058dba6b 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/TextMessageAnalyzer.java @@ -43,7 +43,7 @@ class TextMessageAnalyzer { public static void findTexts() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - List absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NOI18N //get exact file name + List absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NON-NLS //get exact file name for (AbstractFile abstractFile : absFiles) { try { @@ -51,11 +51,11 @@ class TextMessageAnalyzer { ContentUtils.writeToFile(abstractFile, jFile); findTextsInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing text messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding text messages", e); //NOI18N + logger.log(Level.SEVERE, "Error finding text messages", e); //NON-NLS } } @@ -68,17 +68,17 @@ class TextMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "Select address,date,read,type,subject,body FROM sms;"); //NOI18N + "Select address,date,read,type,subject,body FROM sms;"); //NON-NLS String address; // may be phone number, or other addresses @@ -87,15 +87,15 @@ class TextMessageAnalyzer { Integer read; // may be unread = 0, read = 1 String body; //message body while (resultSet.next()) { - address = resultSet.getString("address"); //NOI18N - Long date = Long.valueOf(resultSet.getString("date")) / 1000; //NOI18N + address = resultSet.getString("address"); //NON-NLS + Long date = Long.valueOf(resultSet.getString("date")) / 1000; //NON-NLS - read = resultSet.getInt("read"); //NOI18N - subject = resultSet.getString("subject"); //NOI18N - body = resultSet.getString("body"); //NOI18N + read = resultSet.getInt("read"); //NON-NLS + subject = resultSet.getString("subject"); //NON-NLS + body = resultSet.getString("body"); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create Message artifact and then add attributes from result set. - if (resultSet.getString("type").equals("1")) { //NOI18N + if (resultSet.getString("type").equals("1")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION.getTypeID(), moduleName, "Incoming")); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM.getTypeID(), moduleName, address)); } else { @@ -111,7 +111,7 @@ class TextMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -120,7 +120,7 @@ class TextMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java index d5c584dbce..8977d514be 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/android/WWFMessageAnalyzer.java @@ -44,7 +44,7 @@ class WWFMessageAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='WordsFramework' "); //NOI18N //get exact file names + absFiles = skCase.findAllFilesWhere("name ='WordsFramework' "); //NON-NLS //get exact file names for (AbstractFile abstractFile : absFiles) { try { @@ -53,11 +53,11 @@ class WWFMessageAnalyzer { findWWFMessagesInDB(jFile.toString(), abstractFile); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing WWF messages", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing WWF messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding WWF messages", e); //NOI18N + logger.log(Level.SEVERE, "Error finding WWF messages", e); //NON-NLS } } @@ -74,23 +74,23 @@ class WWFMessageAnalyzer { connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS return; } try { resultSet = statement.executeQuery( - "SELECT message,strftime('%s' ,created_at) as datetime,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); //NOI18N + "SELECT message,strftime('%s' ,created_at) as datetime,user_id,game_id FROM chat_messages ORDER BY game_id DESC, created_at DESC;"); //NON-NLS String message; // WWF Message String user_id; // the ID of the user who sent the message. String game_id; // ID of the game which the the message was sent. while (resultSet.next()) { - message = resultSet.getString("message"); //NOI18N - Long created_at = resultSet.getLong("datetime"); //NOI18N - user_id = resultSet.getString("user_id"); //NOI18N - game_id = resultSet.getString("game_id"); //NOI18N + message = resultSet.getString("message"); //NON-NLS + Long created_at = resultSet.getLong("datetime"); //NON-NLS + user_id = resultSet.getString("user_id"); //NON-NLS + game_id = resultSet.getString("game_id"); //NON-NLS BlackboardArtifact bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create a call log and then add attributes from result set. bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), moduleName, created_at)); @@ -100,7 +100,7 @@ class WWFMessageAnalyzer { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MESSAGE_TYPE.getTypeID(), moduleName, "Words With Friends Message")); } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing WWF messages to the Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing WWF messages to the Blackboard", e); //NON-NLS } finally { try { if (resultSet != null) { @@ -109,7 +109,7 @@ class WWFMessageAnalyzer { statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } From 19dd04a51d9f40aeefb8bfb28554b4c04288ddb3 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Thu, 4 Dec 2014 13:10:47 -0500 Subject: [PATCH 28/84] Extracted 2 more strings that I missed in the first pass. --- .../autopsy/modules/interestingitems/Bundle.properties | 2 ++ .../InterestingItemDefsOptionsPanelController.java | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties index c15d823cca..7a25a4305c 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties @@ -2,6 +2,8 @@ OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=Interesting Files Identifier ingest module. \n\n Identifies interesting files as defined by interesting files rule sets. OpenIDE-Module-Short-Description=Interesting Files Identifier ingest module. OpenIDE-Module-Name=Interesting Files Identifier +OptionsCategory_Name_InterestingItemDefinitions=Interesting Item Definitions +OptionsCategory_Keywords_InterestingItemDefinitions=InterestingItemDefinitions InterestingItemsIdentifierIngestModule.moduleName=Interesting Files Identifier InterestingItemsIdentifierIngestModule.moduleDescription=Identifies interesting items as defined by interesting item rule sets. InterestingItemDefsPanel.newSetButton.text=New Set diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java index c95bffb053..c845bb71e0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java @@ -33,7 +33,6 @@ import org.openide.util.Lookup; keywordsCategory = "InterestingItemDefinitions", position = 5 ) -@org.openide.util.NbBundle.Messages({"OptionsCategory_Name_InterestingItemDefinitions=Interesting Item Definitions", "OptionsCategory_Keywords_InterestingItemDefinitions=InterestingItemDefinitions"}) public final class InterestingItemDefsOptionsPanelController extends OptionsPanelController { private InterestingItemDefsPanel panel; From 73448065d6d3bea18f251e47a3bf7ff00eae8756 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Thu, 4 Dec 2014 13:18:25 -0500 Subject: [PATCH 29/84] Switch NOI18N to NON-NLS for consistency. --- .../autopsy/modules/iOS/CallLogAnalyzer.java | 32 +++++++------- .../autopsy/modules/iOS/ContactAnalyzer.java | 44 +++++++++---------- .../modules/iOS/TextMessageAnalyzer.java | 30 ++++++------- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java index 3514a5d9bc..a3d8cc44ba 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/CallLogAnalyzer.java @@ -49,7 +49,7 @@ class CallLogAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NOI18N //get exact file names + absFiles = skCase.findAllFilesWhere("name ='contacts2.db' OR name ='contacts.db'"); //NON-NLS //get exact file names if (absFiles.isEmpty()) { return; } @@ -61,11 +61,11 @@ class CallLogAnalyzer { fileId = AF.getId(); findCallLogsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Call logs", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Call logs", e); //NOI18N + logger.log(Level.SEVERE, "Error finding Call logs", e); //NON-NLS } } @@ -74,11 +74,11 @@ class CallLogAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS } Case currentCase = Case.getCurrentCase(); @@ -87,7 +87,7 @@ class CallLogAnalyzer { AbstractFile f = skCase.getAbstractFileById(fId); try { resultSet = statement.executeQuery( - "SELECT number,date,duration,type, name FROM calls ORDER BY date DESC;"); //NOI18N + "SELECT number,date,duration,type, name FROM calls ORDER BY date DESC;"); //NON-NLS BlackboardArtifact bba; String name; // name of person dialed or called. null if unregistered @@ -97,14 +97,14 @@ class CallLogAnalyzer { String type; // 1 incoming, 2 outgoing, 3 missed while (resultSet.next()) { - name = resultSet.getString("name"); //NOI18N - number = resultSet.getString("number"); //NOI18N - duration = resultSet.getString("duration"); //NOI18N - date = resultSet.getString("date"); //NOI18N - type = resultSet.getString("type"); //NOI18N + name = resultSet.getString("name"); //NON-NLS + number = resultSet.getString("number"); //NON-NLS + duration = resultSet.getString("duration"); //NON-NLS + date = resultSet.getString("date"); //NON-NLS + type = resultSet.getString("type"); //NON-NLS bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); //create a call log and then add attributes from result set. - if(type.equalsIgnoreCase("outgoing")) { //NOI18N + if(type.equalsIgnoreCase("outgoing")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO.getTypeID(), moduleName, number)); } else { /// Covers INCOMING and MISSED @@ -117,18 +117,18 @@ class CallLogAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NON-NLS } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing the database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing the database", e); //NON-NLS } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Call logs to the Blackboard", e); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java index a119666993..becafbafc0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/ContactAnalyzer.java @@ -56,7 +56,7 @@ class ContactAnalyzer { List absFiles; try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name LIKE '%call_history%' "); //NOI18N //get exact file names + absFiles = skCase.findAllFilesWhere("name LIKE '%call_history%' "); //NON-NLS //get exact file names if (absFiles.isEmpty()) { //asdfkjasfakljsdfhlaksdjfhasdlkjf return; } @@ -71,11 +71,11 @@ class ContactAnalyzer { fileId = AF.getId(); //findContactsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Contacts", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding Contacts", e); //NOI18N + logger.log(Level.SEVERE, "Error finding Contacts", e); //NON-NLS } } @@ -90,11 +90,11 @@ class ContactAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS } Case currentCase = Case.getCurrentCase(); @@ -105,13 +105,13 @@ class ContactAnalyzer { // get display_name, mimetype(email or phone number) and data1 (phonenumber or email address depending on mimetype) //sorted by name, so phonenumber/email would be consecutive for a person if they exist. resultSet = statement.executeQuery( - "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NOI18N - + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NOI18N - + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NOI18N - + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NOI18N - + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NOI18N - + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NOI18N - + "ORDER BY name_raw_contact.display_name ASC;"); //NOI18N + "SELECT mimetype,data1, name_raw_contact.display_name AS display_name \n" //NON-NLS + + "FROM raw_contacts JOIN contacts ON (raw_contacts.contact_id=contacts._id) \n" //NON-NLS + + "JOIN raw_contacts AS name_raw_contact ON(name_raw_contact_id=name_raw_contact._id) " //NON-NLS + + "LEFT OUTER JOIN data ON (data.raw_contact_id=raw_contacts._id) \n" //NON-NLS + + "LEFT OUTER JOIN mimetypes ON (data.mimetype_id=mimetypes._id) \n" //NON-NLS + + "WHERE mimetype = 'vnd.android.cursor.item/phone_v2' OR mimetype = 'vnd.android.cursor.item/email_v2'\n" //NON-NLS + + "ORDER BY name_raw_contact.display_name ASC;"); //NON-NLS BlackboardArtifact bba; bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); @@ -120,15 +120,15 @@ class ContactAnalyzer { String mimetype; // either phone or email String data1; // the phone number or email while (resultSet.next()) { - name = resultSet.getString("display_name"); //NOI18N - data1 = resultSet.getString("data1"); //NOI18N - mimetype = resultSet.getString("mimetype"); //NOI18N + name = resultSet.getString("display_name"); //NON-NLS + data1 = resultSet.getString("data1"); //NON-NLS + mimetype = resultSet.getString("mimetype"); //NON-NLS // System.out.println(resultSet.getString("data1") + resultSet.getString("mimetype") + resultSet.getString("display_name")); //Test code if (name.equals(oldName) == false) { bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, name)); } - if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NOI18N + if (mimetype.equals("vnd.android.cursor.item/phone_v2")) { //NON-NLS bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getTypeID(), moduleName, data1)); } else { bba.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getTypeID(), moduleName, data1)); @@ -137,18 +137,18 @@ class ContactAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NON-NLS } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing Contacts to Blackboard", e); //NON-NLS } } @@ -186,13 +186,13 @@ class ContactAnalyzer { ostream.write(c); } } catch (IOException e) { - System.out.println("Error: " + e.getMessage()); //NOI18N + System.out.println("Error: " + e.getMessage()); //NON-NLS } finally { try { istream.close(); ostream.close(); } catch (IOException e) { - System.out.println("File did not close"); //NOI18N + System.out.println("File did not close"); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java b/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java index 18dae2c61c..04d67b78f7 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java +++ b/Core/src/org/sleuthkit/autopsy/modules/iOS/TextMessageAnalyzer.java @@ -49,7 +49,7 @@ class TextMessageAnalyzer { void findTexts() { try { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); - absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NOI18N //get exact file name + absFiles = skCase.findAllFilesWhere("name ='mmssms.db'"); //NON-NLS //get exact file name if (absFiles.isEmpty()) { return; } @@ -61,11 +61,11 @@ class TextMessageAnalyzer { fileId = AF.getId(); findTextsInDB(dbPath, fileId); } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing text messages", e); //NON-NLS } } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error finding text messages", e); //NOI18N + logger.log(Level.SEVERE, "Error finding text messages", e); //NON-NLS } } @@ -74,11 +74,11 @@ class TextMessageAnalyzer { return; } try { - Class.forName("org.sqlite.JDBC"); //NOI18N //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NOI18N + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + DatabasePath); //NON-NLS statement = connection.createStatement(); } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database", e); //NOI18N + logger.log(Level.SEVERE, "Error opening database", e); //NON-NLS } Case currentCase = Case.getCurrentCase(); @@ -87,7 +87,7 @@ class TextMessageAnalyzer { AbstractFile f = skCase.getAbstractFileById(fId); try { resultSet = statement.executeQuery( - "Select address,date,type,subject,body FROM sms;"); //NOI18N + "Select address,date,type,subject,body FROM sms;"); //NON-NLS BlackboardArtifact bba; String address; // may be phone number, or other addresses @@ -96,11 +96,11 @@ class TextMessageAnalyzer { String subject;//message subject String body; //message body while (resultSet.next()) { - address = resultSet.getString("address"); //NOI18N - date = resultSet.getString("date"); //NOI18N - type = resultSet.getString("type"); //NOI18N - subject = resultSet.getString("subject"); //NOI18N - body = resultSet.getString("body"); //NOI18N + address = resultSet.getString("address"); //NON-NLS + date = resultSet.getString("date"); //NON-NLS + type = resultSet.getString("type"); //NON-NLS + subject = resultSet.getString("subject"); //NON-NLS + body = resultSet.getString("body"); //NON-NLS bba = f.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); //create Message artifact and then add attributes from result set. @@ -122,18 +122,18 @@ class TextMessageAnalyzer { } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NON-NLS } finally { try { resultSet.close(); statement.close(); connection.close(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error closing database", e); //NOI18N + logger.log(Level.SEVERE, "Error closing database", e); //NON-NLS } } } catch (Exception e) { - logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NOI18N + logger.log(Level.SEVERE, "Error parsing text messages to Blackboard", e); //NON-NLS } } From a75ebe8028a9bc69dfef101e79b9b16bd7d796c7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 4 Dec 2014 18:46:17 -0500 Subject: [PATCH 30/84] Commit version of file types manager with fine grained-control, simpler will be better --- .../modules/filetypeid/Bundle.properties | 30 ++-- .../modules/filetypeid/Bundle_ja.properties | 20 +-- ...orm => FileTypeIdGlobalSettingsPanel.form} | 26 ++-- ...ava => FileTypeIdGlobalSettingsPanel.java} | 30 ++-- ... => FileTypeIdIngestJobSettingsPanel.form} | 4 +- ... => FileTypeIdIngestJobSettingsPanel.java} | 8 +- .../filetypeid/FileTypeIdModuleFactory.java | 4 +- .../FileTypeIdOptionsPanelController.java | 6 +- .../UserDefinedFileTypesManager.java | 144 ++++++++++-------- 9 files changed, 145 insertions(+), 127 deletions(-) rename Core/src/org/sleuthkit/autopsy/modules/filetypeid/{FileTypeIdSettingsPanel.form => FileTypeIdGlobalSettingsPanel.form} (87%) rename Core/src/org/sleuthkit/autopsy/modules/filetypeid/{FileTypeIdSettingsPanel.java => FileTypeIdGlobalSettingsPanel.java} (90%) rename Core/src/org/sleuthkit/autopsy/modules/filetypeid/{FileTypeIdModuleSettingsPanel.form => FileTypeIdIngestJobSettingsPanel.form} (87%) rename Core/src/org/sleuthkit/autopsy/modules/filetypeid/{FileTypeIdModuleSettingsPanel.java => FileTypeIdIngestJobSettingsPanel.java} (89%) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 8df981383f..4481e95ab3 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -1,5 +1,4 @@ OpenIDE-Module-Name=FileTypeId -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText=Depending on how many files have known hashes, checking this box will improve the speed of file type identification. FileTypeIdIngestModule.moduleName.text=File Type Identification FileTypeIdIngestModule.moduleDesc.text=Matches file types based on binary signatures. FileTypeIdIngestModule.complete.totalProcTime=Total Processing Time @@ -7,17 +6,18 @@ FileTypeIdIngestModule.complete.totalFiles=Total Files Processed FileTypeIdIngestModule.complete.srvMsg.text=File Type Id Results FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings FileTypeIdModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof FileTypeIdModuleSettings -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) -FileTypeIdSettingsPanel.mimeTypeTextField.text= -FileTypeIdSettingsPanel.mimeTypeLabel.text=Mime Type -FileTypeIdSettingsPanel.signatureTypeLabel.text=Signature Type -FileTypeIdSettingsPanel.signatureLabel.text=Signature -FileTypeIdSettingsPanel.offsetLabel.text=Offset -FileTypeIdSettingsPanel.offsetTextField.text= -FileTypeIdSettingsPanel.hexPrefixLabel.text=0x -FileTypeIdSettingsPanel.jTextArea1.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. -FileTypeIdSettingsPanel.postHitCheckBox.text=Post interesting file hit when found -FileTypeIdSettingsPanel.newTypeButton.text=New Type -FileTypeIdSettingsPanel.deleteTypeButton.text=DeleteType -FileTypeIdSettingsPanel.saveTypeButton.text=Save Type -FileTypeIdSettingsPanel.signatureTextField.text= +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText=Depending on how many files have known hashes, checking this box will improve the speed of file type identification. +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) +FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text=0x +FileTypeIdGlobalSettingsPanel.saveTypeButton.text=Save Type +FileTypeIdGlobalSettingsPanel.deleteTypeButton.text=DeleteType +FileTypeIdGlobalSettingsPanel.newTypeButton.text=New Type +FileTypeIdGlobalSettingsPanel.offsetTextField.text= +FileTypeIdGlobalSettingsPanel.offsetLabel.text=Offset +FileTypeIdGlobalSettingsPanel.postHitCheckBox.text=Post interesting file hit when found +FileTypeIdGlobalSettingsPanel.signatureTextField.text= +FileTypeIdGlobalSettingsPanel.jTextArea1.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. +FileTypeIdGlobalSettingsPanel.signatureTypeLabel.text=Signature Type +FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text= +FileTypeIdGlobalSettingsPanel.signatureLabel.text=Signature +FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text=Mime Type diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties index a12d0e7cd8..ee53301d77 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties @@ -1,10 +1,10 @@ -OpenIDE-Module-Name=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText=\u65E2\u77E5\u306E\u30CF\u30C3\u30B7\u30E5\u5024\u3092\u6301\u3064\u30D5\u30A1\u30A4\u30EB\u6570\u306B\u3088\u3063\u3066\u306F\u3001\u3053\u306E\u30DC\u30C3\u30AF\u30B9\u3092\u9078\u629E\u3059\u308B\u306E\u306B\u3088\u308A\u3001\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A\u3092\u52A0\u901F\u3057\u307E\u3059\u3002 -FileTypeIdIngestModule.moduleName.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A -FileTypeIdIngestModule.moduleDesc.text=\u30D0\u30A4\u30CA\u30EA\u7F72\u540D\u306B\u57FA\u3065\u3044\u3066\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u3092\u4E00\u81F4\u3059\u308B\u3002 -FileTypeIdIngestModule.complete.totalProcTime=\u5408\u8A08\u51E6\u7406\u6642\u9593 -FileTypeIdIngestModule.complete.totalFiles=\u5408\u8A08\u51E6\u7406\u30D5\u30A1\u30A4\u30EB\u6570 -FileTypeIdIngestModule.complete.srvMsg.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u7279\u5B9A\u306E\u7D50\u679C -FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\uFF08NSRL\uFF09\u3092\u30B9\u30AD\u30C3\u30D7 -FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=\u8A2D\u5B9A\u3092\u884C\u3046\u70BA\u306E\u60F3\u5B9A\u3055\u308C\u308B\u5F15\u6570\u306Finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 -FileTypeIdModuleFactory.createFileIngestModule.exception.msg=\u8A2D\u5B9A\u3092\u884C\u3046\u70BA\u306E\u60F3\u5B9A\u3055\u308C\u308B\u5F15\u6570\u306Finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 \ No newline at end of file +OpenIDE-Module-Name=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a +FileTypeIdIngestModule.moduleName.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a +FileTypeIdIngestModule.moduleDesc.text=\u30d0\u30a4\u30ca\u30ea\u7f72\u540d\u306b\u57fa\u3065\u3044\u3066\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u3092\u4e00\u81f4\u3059\u308b\u3002 +FileTypeIdIngestModule.complete.totalProcTime=\u5408\u8a08\u51e6\u7406\u6642\u9593 +FileTypeIdIngestModule.complete.totalFiles=\u5408\u8a08\u51e6\u7406\u30d5\u30a1\u30a4\u30eb\u6570 +FileTypeIdIngestModule.complete.srvMsg.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u7279\u5b9a\u306e\u7d50\u679c +FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg=\u8a2d\u5b9a\u3092\u884c\u3046\u70ba\u306e\u60f3\u5b9a\u3055\u308c\u308b\u5f15\u6570\u306finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 +FileTypeIdModuleFactory.createFileIngestModule.exception.msg=\u8a2d\u5b9a\u3092\u884c\u3046\u70ba\u306e\u60f3\u5b9a\u3055\u308c\u308b\u5f15\u6570\u306finstanceof FileTypeIdModuleSettings\u3067\u3059\u3002 +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText=\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u6301\u3064\u30d5\u30a1\u30a4\u30eb\u6570\u306b\u3088\u3063\u3066\u306f\u3001\u3053\u306e\u30dc\u30c3\u30af\u30b9\u3092\u9078\u629e\u3059\u308b\u306e\u306b\u3088\u308a\u3001\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a\u3092\u52a0\u901f\u3057\u307e\u3059\u3002 +FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text=\u65e2\u77e5\u30d5\u30a1\u30a4\u30eb\uff08NSRL\uff09\u3092\u30b9\u30ad\u30c3\u30d7 diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form similarity index 87% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index d8ad98b738..fd6d2250d3 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -178,70 +178,70 @@ - + - + - + - + - + - + - + - + - + - + @@ -259,7 +259,7 @@ - + @@ -280,7 +280,7 @@ - + @@ -290,7 +290,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java similarity index 90% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 595a32397e..e9b0060653 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -30,7 +30,7 @@ import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.User /** * A panel to allow a user to make custom file type definitions. */ -final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { +final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { private final HashMap fileTypes = new HashMap<>(); private final HashMap changedFileTypes = new HashMap<>(); @@ -39,7 +39,7 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel impl /** * Creates a panel to allow a user to make custom file type definitions. */ - FileTypeIdSettingsPanel() { + FileTypeIdGlobalSettingsPanel() { initComponents(); } @@ -139,40 +139,40 @@ final class FileTypeIdSettingsPanel extends IngestModuleGlobalSettingsPanel impl jSeparator1.setOrientation(javax.swing.SwingConstants.VERTICAL); - org.openide.awt.Mnemonics.setLocalizedText(mimeTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.mimeTypeLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(mimeTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text")); // NOI18N - mimeTypeTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.mimeTypeTextField.text")); // NOI18N + mimeTypeTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(signatureTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.signatureTypeLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(signatureTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureTypeLabel.text")); // NOI18N - signatureTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.signatureTextField.text")); // NOI18N + signatureTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureTextField.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(offsetLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.offsetLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(offsetLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.offsetLabel.text")); // NOI18N - offsetTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.offsetTextField.text")); // NOI18N + offsetTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.offsetTextField.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(newTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.newTypeButton.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(newTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.newTypeButton.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(deleteTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.deleteTypeButton.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(deleteTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.deleteTypeButton.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(saveTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.saveTypeButton.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(saveTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.saveTypeButton.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(hexPrefixLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.hexPrefixLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(hexPrefixLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text")); // NOI18N signatureTypeComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "Bytes (Hex)", "String", " " })); - org.openide.awt.Mnemonics.setLocalizedText(signatureLabel, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.signatureLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(signatureLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureLabel.text")); // NOI18N jTextArea1.setEditable(false); jTextArea1.setColumns(20); jTextArea1.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N jTextArea1.setLineWrap(true); jTextArea1.setRows(5); - jTextArea1.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.jTextArea1.text")); // NOI18N + jTextArea1.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.jTextArea1.text")); // NOI18N jTextArea1.setWrapStyleWord(true); jScrollPane2.setViewportView(jTextArea1); - org.openide.awt.Mnemonics.setLocalizedText(postHitCheckBox, org.openide.util.NbBundle.getMessage(FileTypeIdSettingsPanel.class, "FileTypeIdSettingsPanel.postHitCheckBox.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(postHitCheckBox, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.postHitCheckBox.text")); // NOI18N javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form similarity index 87% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form index 60dfbc7cd3..7a24e05f05 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form @@ -38,10 +38,10 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java similarity index 89% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java index bf86217a50..f31380379d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java @@ -25,13 +25,13 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; * UI component used to set ingest job options for file type identifier ingest * modules. */ -final class FileTypeIdModuleSettingsPanel extends IngestModuleIngestJobSettingsPanel { +final class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPanel { private final FileTypeIdModuleSettings settings; // NOTE: This was declared public, but was inaccessible because the class is // not public - FileTypeIdModuleSettingsPanel(FileTypeIdModuleSettings settings) { + FileTypeIdIngestJobSettingsPanel(FileTypeIdModuleSettings settings) { this.settings = settings; initComponents(); customizeComponents(); @@ -61,8 +61,8 @@ final class FileTypeIdModuleSettingsPanel extends IngestModuleIngestJobSettingsP skipKnownCheckBox = new javax.swing.JCheckBox(); skipKnownCheckBox.setSelected(true); - skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdModuleSettingsPanel.class, "FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text")); // NOI18N - skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdModuleSettingsPanel.class, "FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText")); // NOI18N + skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text")); // NOI18N + skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText")); // NOI18N skipKnownCheckBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { skipKnownCheckBoxActionPerformed(evt); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java index 73d1e140e5..dbd54351cc 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java @@ -82,7 +82,7 @@ public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { */ @Override public IngestModuleGlobalSettingsPanel getGlobalSettingsPanel() { - return new FileTypeIdSettingsPanel(); + return new FileTypeIdGlobalSettingsPanel(); } /** @@ -111,7 +111,7 @@ public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { throw new IllegalArgumentException(NbBundle.getMessage(this.getClass(), "FileTypeIdModuleFactory.getIngestJobSettingsPanel.exception.msg")); } - return new FileTypeIdModuleSettingsPanel((FileTypeIdModuleSettings) settings); + return new FileTypeIdIngestJobSettingsPanel((FileTypeIdModuleSettings) settings); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java index 44c7730c1a..45d24b9db4 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java @@ -22,7 +22,7 @@ import org.openide.util.Lookup; @org.openide.util.NbBundle.Messages({"OptionsCategory_Name_FileTypeId=FileTypeId", "OptionsCategory_Keywords_FileTypeId=FileTypeId"}) public final class FileTypeIdOptionsPanelController extends OptionsPanelController { - private FileTypeIdSettingsPanel panel; + private FileTypeIdGlobalSettingsPanel panel; private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); private boolean changed; @@ -77,9 +77,9 @@ public final class FileTypeIdOptionsPanelController extends OptionsPanelControll pcs.removePropertyChangeListener(l); } - private FileTypeIdSettingsPanel getPanel() { + private FileTypeIdGlobalSettingsPanel getPanel() { if (panel == null) { - panel = new FileTypeIdSettingsPanel(); + panel = new FileTypeIdGlobalSettingsPanel(); } return panel; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index ee0f601bf2..c3f177a4c9 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,21 +48,35 @@ import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; final class UserDefinedFileTypesManager { private static final Logger logger = Logger.getLogger(UserDefinedFileTypesManager.class.getName()); - private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; // NON-NLS - private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; // NON-NLS - private static final String FILE_TYPES_TAG_NAME = "filetypes"; // NON-NLS - private static final String FILE_TYPE_TAG_NAME = "filetype"; // NON-NLS - private static final String ALERT_ATTRIBUTE = "alert"; // NON-NLS - private static final String TYPE_NAME_TAG_NAME = "typename"; // NON-NLS - private static final String SIGNATURE_TAG_NAME = "signature"; // NON-NLS - private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; // NON-NLS - private static final String BYTES_TAG_NAME = "bytes"; // NON-NLS - private static final String OFFSET_TAG_NAME = "offset"; // NON-NLS - private static final String ENCODING = "UTF-8"; //NON-NLS + private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; //NON-NLS + private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; //NON-NLS + private static final String FILE_TYPES_TAG_NAME = "filetypes"; //NON-NLS + private static final String FILE_TYPE_TAG_NAME = "filetype"; //NON-NLS + private static final String ALERT_ATTRIBUTE = "alert"; //NON-NLS + private static final String TYPE_NAME_TAG_NAME = "typename"; //NON-NLS + private static final String SIGNATURE_TAG_NAME = "signature"; //NON-NLS + private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; //NON-NLS + private static final String BYTES_TAG_NAME = "bytes"; //NON-NLS + private static final String OFFSET_TAG_NAME = "offset"; //NON-NLS + private static final String ENCODING_FOR_XML_FILE = "UTF-8"; //NON-NLS private static final String ASCII_ENCODING = "US-ASCII"; //NON-NLS private static UserDefinedFileTypesManager instance; - private final Map predefinedFileTypes = new HashMap<>(); + + /** + * User-defined file types to be persisted to the user-defined file type + * definitions file are stored in this mapping of file type names to file + * types. Access to this map is guarded by the intrinsic lock of the + * user-defined file types manager for thread-safety. + */ private final Map userDefinedFileTypes = new HashMap<>(); + + /** + * The combined set of user-defined file types and file types predefined by + * Autopsy are stored in this mapping of file type names to file types. This + * is the current working set of file types. Access to this map is guarded + * by the intrinsic lock of the user-defined file types manager for + * thread-safety. + */ private final Map fileTypes = new HashMap<>(); /** @@ -70,10 +85,10 @@ final class UserDefinedFileTypesManager { * @return A singleton user-defined file types manager. */ synchronized static UserDefinedFileTypesManager getInstance() { - if (instance == null) { - instance = new UserDefinedFileTypesManager(); + if (UserDefinedFileTypesManager.instance == null) { + UserDefinedFileTypesManager.instance = new UserDefinedFileTypesManager(); } - return instance; + return UserDefinedFileTypesManager.instance; } /** @@ -90,7 +105,7 @@ final class UserDefinedFileTypesManager { * names to predefined file types. */ private void loadPredefinedFileTypes() { - // RJCTODO: Remove + // RJCTODO: Remove test type /** * Create a file type that should match $MBR in Small2 image. */ @@ -99,11 +114,11 @@ final class UserDefinedFileTypesManager { /** * Create a file type that should match test.txt in the Small2 image. */ - // RJCTODO: Remove + // RJCTODO: Remove test type try { - this.fileTypes.put("predefinedASCII", new FileType("predefinedASCII", new Signature("hello".getBytes(ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true)); + this.fileTypes.put("predefinedASCII", new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true)); } catch (UnsupportedEncodingException ex) { - logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS + UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS } try { @@ -129,9 +144,13 @@ final class UserDefinedFileTypesManager { // catch (IndexOutOfBoundsException e) { // // do nothing // } - this.predefinedFileTypes.put("text/xml", new FileType("text/xml", new Signature(" newFileTypes = new ArrayList<>(); - newFileTypes.add(fileType); - addFileTypes(newFileTypes); + this.addFileTypes(Collections.singletonList(fileType)); } /** - * Adds a set of new file types, overwriting any file types with the same - * type names that already exist. + * Adds a collection of new user-defined file types, overwriting any + * existing file types with the same type names. * * @param newFileTypes The file types to add. * @throws * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException */ synchronized void addFileTypes(Collection newFileTypes) throws UserDefinedFileTypesException { + /** + * It is safe to hold references to client-constructed file type objects + * because they are immutable. + */ for (FileType fileType : newFileTypes) { - userDefinedFileTypes.put(fileType.getTypeName(), fileType); - fileTypes.put(fileType.getTypeName(), fileType); + this.userDefinedFileTypes.put(fileType.getTypeName(), fileType); + this.fileTypes.put(fileType.getTypeName(), fileType); } - saveUserDefinedTypes(); + this.saveUserDefinedTypes(); } /** - * Deletes a file type. + * Deletes a user-defined file type. * * @param fileType The file type to delete. * @throws * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException */ synchronized void deleteFileType(FileType fileType) throws UserDefinedFileTypesException { - List deletedFileTypes = new ArrayList<>(); - deletedFileTypes.add(fileType); - deleteFileTypes(deletedFileTypes); + this.deleteFileTypes(Collections.singletonList(fileType)); } /** - * Deletes a set of file types. + * Deletes a set of user-defined file types. * * @param deletedFileTypes The file types to delete. * @throws @@ -227,21 +244,21 @@ final class UserDefinedFileTypesManager { */ synchronized void deleteFileTypes(Collection deletedFileTypes) throws UserDefinedFileTypesException { for (FileType fileType : deletedFileTypes) { - userDefinedFileTypes.remove(fileType.getTypeName(), fileType); - fileTypes.remove(fileType.getTypeName(), fileType); + this.userDefinedFileTypes.remove(fileType.getTypeName(), fileType); + this.fileTypes.remove(fileType.getTypeName(), fileType); } - saveUserDefinedTypes(); + this.saveUserDefinedTypes(); } /** - * Persist the user-defined file type definitions. + * Persists the user-defined file type definitions. * * @throws * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException */ private void saveUserDefinedTypes() throws UserDefinedFileTypesException { - String filePath = getFileTypeDefinitionsFilePath(USER_DEFINED_TYPE_DEFINITIONS_FILE); - writeFileTypes(userDefinedFileTypes.values(), filePath); + String filePath = UserDefinedFileTypesManager.getFileTypeDefinitionsFilePath(UserDefinedFileTypesManager.USER_DEFINED_TYPE_DEFINITIONS_FILE); + UserDefinedFileTypesManager.writeFileTypes(this.userDefinedFileTypes.values(), filePath); } /** @@ -253,10 +270,10 @@ final class UserDefinedFileTypesManager { */ private static void writeFileTypes(Collection fileTypes, String filePath) throws UserDefinedFileTypesException { try { - XMLWriter.writeFileTypes(fileTypes, filePath); + UserDefinedFileTypesManager.XMLWriter.writeFileTypes(fileTypes, filePath); } catch (ParserConfigurationException | IOException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); - throw new UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message + throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message } } @@ -291,9 +308,9 @@ final class UserDefinedFileTypesManager { Element fileTypeElem = UserDefinedFileTypesManager.XMLWriter.createFileTypeElement(fileType, doc); fileTypesElem.appendChild(fileTypeElem); } - if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypesManager.ENCODING, doc)) { + if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypesManager.ENCODING_FOR_XML_FILE, doc)) { // RJCTODO: If time permits add XMLUtil that properly throws and deprecate this one - throw new IOException("Error saving user defined file types, see log for details"); // NON-NLS + throw new IOException("Error saving user defined file types, see log for details"); //NON-NLS } } @@ -451,17 +468,18 @@ final class UserDefinedFileTypesManager { if (!textContent.isEmpty()) { return textContent; } else { - throw new InvalidXMLException("File type " + tagName + " child element missing text content"); // NON-NLS + throw new InvalidXMLException("File type " + tagName + " child element missing text content"); //NON-NLS } } else { - throw new InvalidXMLException("File type element missing " + tagName + " child element"); // NON-NLS + throw new InvalidXMLException("File type element missing " + tagName + " child element"); //NON-NLS } } /** - * RJCTODO + * Used for exceptions when parsing user-defined types XML elements and + * attributes. */ - static class InvalidXMLException extends Exception { + private static class InvalidXMLException extends Exception { InvalidXMLException(String message) { super(message); From a9bc43e320bdf80eb2def997da4f0b7fd1e0dd2f Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 5 Dec 2014 00:45:01 -0500 Subject: [PATCH 31/84] Work on user-defined file types UI --- .../modules/filetypeid/Bundle.properties | 2 +- .../FileTypeIdGlobalSettingsPanel.form | 52 ++--- .../FileTypeIdGlobalSettingsPanel.java | 179 +++++++++++++----- .../UserDefinedFileTypeIdentifier.java | 11 +- .../UserDefinedFileTypesManager.java | 152 +++++++-------- 5 files changed, 241 insertions(+), 155 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 4481e95ab3..2d18364283 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -9,7 +9,6 @@ FileTypeIdModuleFactory.createFileIngestModule.exception.msg=Expected settings a FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText=Depending on how many files have known hashes, checking this box will improve the speed of file type identification. FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text=0x -FileTypeIdGlobalSettingsPanel.saveTypeButton.text=Save Type FileTypeIdGlobalSettingsPanel.deleteTypeButton.text=DeleteType FileTypeIdGlobalSettingsPanel.newTypeButton.text=New Type FileTypeIdGlobalSettingsPanel.offsetTextField.text= @@ -21,3 +20,4 @@ FileTypeIdGlobalSettingsPanel.signatureTypeLabel.text=Signature Type FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text= FileTypeIdGlobalSettingsPanel.signatureLabel.text=Signature FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text=Mime Type +FileTypeIdGlobalSettingsPanel.saveTypeButton.text=Save Type diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index fd6d2250d3..49ae296a57 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -35,7 +35,6 @@ - @@ -43,18 +42,17 @@ - - + + - - + @@ -78,22 +76,17 @@ - - - - - - - - - - - + + + + + + - + @@ -141,14 +134,14 @@     - +     - +     @@ -167,6 +160,9 @@ + + +     @@ -223,6 +219,9 @@ + + + @@ -230,6 +229,9 @@ + + + @@ -237,6 +239,9 @@ + + + @@ -248,13 +253,12 @@ - - - - - + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index e9b0060653..aaafa786eb 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -18,23 +18,30 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; -import java.util.HashMap; +import java.util.Map; +import javax.swing.DefaultComboBoxModel; import javax.swing.DefaultListModel; +import javax.swing.JOptionPane; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; -import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException; /** * A panel to allow a user to make custom file type definitions. */ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { - private final HashMap fileTypes = new HashMap<>(); - private final HashMap changedFileTypes = new HashMap<>(); - private final HashMap deletedFileTypes = new HashMap<>(); + private static final String RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM = "Bytes (Hex)"; // RJCTODO: Bundle + private static final String ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM = "String (ASCII)"; // RJCTODO: Bundle + + /** + * This mapping of file type names to file types is used to hold the types + * displayed in the file types list component. + */ + private Map fileTypes; /** * Creates a panel to allow a user to make custom file type definitions. @@ -56,19 +63,38 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane */ @Override public void load() { - fileTypes.clear(); + /** + * Get the user-defined file types and set up a list model for the file + * types list component. + */ + this.fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); DefaultListModel fileTypesListModel = new DefaultListModel<>(); - for (FileType fileType : UserDefinedFileTypesManager.getInstance().getFileTypes()) { - fileTypes.put(fileType.getTypeName(), fileType); + for (FileType fileType : this.fileTypes.values()) { fileTypesListModel.addElement(fileType.getTypeName()); } - typesList.setModel(fileTypesListModel); + this.typesList.setModel(fileTypesListModel); - typesList.addListSelectionListener(new TypesListSelectionListener()); + /** + * Add a selection listener to populate the file type details + * display/edit components. + */ + this.typesList.addListSelectionListener(new TypesListSelectionListener()); + /** + * If there is at least one user-defined file type, select it the file + * types list component. + */ if (!fileTypesListModel.isEmpty()) { - typesList.setSelectedIndex(0); + this.typesList.setSelectedIndex(0); } + + /** + * Make a model for the signature type combo box component. + */ + DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); } /** @@ -77,10 +103,8 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane @Override public void store() { try { - UserDefinedFileTypesManager fileTypesManager = UserDefinedFileTypesManager.getInstance(); - fileTypesManager.addFileTypes(changedFileTypes.values()); - fileTypesManager.deleteFileTypes(deletedFileTypes.values()); - } catch (UserDefinedFileTypesException ex) { + UserDefinedFileTypesManager.getInstance().setUserDefinedFileTypes(this.fileTypes); + } catch (UserDefinedFileTypesManager.UserDefinedFileTypesException ex) { // RJCTODO } } @@ -90,22 +114,36 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane @Override public void valueChanged(ListSelectionEvent e) { if (e.getValueIsAdjusting() == false) { - if (typesList.getSelectedIndex() == -1) { - deleteTypeButton.setEnabled(false); - + if (FileTypeIdGlobalSettingsPanel.this.typesList.getSelectedIndex() == -1) { + FileTypeIdGlobalSettingsPanel.this.deleteTypeButton.setEnabled(false); } else { - String typeName = (String) typesList.getSelectedValue(); - FileType fileType = fileTypes.get(typeName); + String typeName = FileTypeIdGlobalSettingsPanel.this.typesList.getSelectedValue(); + FileType fileType = FileTypeIdGlobalSettingsPanel.this.fileTypes.get(typeName); Signature signature = fileType.getSignature(); - mimeTypeTextField.setText(typeName); - offsetTextField.setText(Long.toString(signature.getOffset())); - deleteTypeButton.setEnabled(true); + FileTypeIdGlobalSettingsPanel.this.mimeTypeTextField.setText(typeName); + FileType.Signature.Type sigType = fileType.getSignature().getType(); + FileTypeIdGlobalSettingsPanel.this.signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + FileTypeIdGlobalSettingsPanel.this.offsetTextField.setText(Long.toString(signature.getOffset())); + FileTypeIdGlobalSettingsPanel.this.postHitCheckBox.setSelected(fileType.alertOnMatch()); + FileTypeIdGlobalSettingsPanel.this.deleteTypeButton.setEnabled(true); } } } } - // RJCTODO: Consider fixing OptinsPanel interface or writing story + /** + * RJCTODO + */ + private void clearTypeDetailsComponents() { + this.typesList.setSelectedIndex(-1); + this.mimeTypeTextField.setText(""); //NON-NLS // RJCTODO: Default name? + this.signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + this.signatureTextField.setText(""); //NON-NLS + this.offsetTextField.setText(""); //NON-NLS + this.postHitCheckBox.setSelected(false); + } + + // RJCTODO: Consider fixing OptionsPanel interface or writing story /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -117,7 +155,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane buttonGroup1 = new javax.swing.ButtonGroup(); typesScrollPane = new javax.swing.JScrollPane(); - typesList = new javax.swing.JList(); + typesList = new javax.swing.JList(); jSeparator1 = new javax.swing.JSeparator(); mimeTypeLabel = new javax.swing.JLabel(); mimeTypeTextField = new javax.swing.JTextField(); @@ -129,7 +167,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane deleteTypeButton = new javax.swing.JButton(); saveTypeButton = new javax.swing.JButton(); hexPrefixLabel = new javax.swing.JLabel(); - signatureTypeComboBox = new javax.swing.JComboBox(); + signatureTypeComboBox = new javax.swing.JComboBox(); signatureLabel = new javax.swing.JLabel(); jScrollPane2 = new javax.swing.JScrollPane(); jTextArea1 = new javax.swing.JTextArea(); @@ -152,15 +190,28 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane offsetTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.offsetTextField.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(newTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.newTypeButton.text")); // NOI18N + newTypeButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + newTypeButtonActionPerformed(evt); + } + }); org.openide.awt.Mnemonics.setLocalizedText(deleteTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.deleteTypeButton.text")); // NOI18N + deleteTypeButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + deleteTypeButtonActionPerformed(evt); + } + }); org.openide.awt.Mnemonics.setLocalizedText(saveTypeButton, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.saveTypeButton.text")); // NOI18N + saveTypeButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + saveTypeButtonActionPerformed(evt); + } + }); org.openide.awt.Mnemonics.setLocalizedText(hexPrefixLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text")); // NOI18N - signatureTypeComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "Bytes (Hex)", "String", " " })); - org.openide.awt.Mnemonics.setLocalizedText(signatureLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureLabel.text")); // NOI18N jTextArea1.setEditable(false); @@ -190,25 +241,23 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(37, 37, 37) - .addComponent(saveTypeButton) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addComponent(saveTypeButton)) .addGroup(layout.createSequentialGroup() .addGap(18, 18, 18) .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() .addComponent(signatureTypeLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 82, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(0, 0, Short.MAX_VALUE)) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 82, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addComponent(signatureLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(hexPrefixLabel) .addGap(2, 2, 2)) .addGroup(layout.createSequentialGroup() @@ -224,15 +273,12 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 181, javax.swing.GroupLayout.PREFERRED_SIZE)))) .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addGap(6, 6, 6) - .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGroup(layout.createSequentialGroup() - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(postHitCheckBox))) - .addGap(0, 0, Short.MAX_VALUE))) - .addContainerGap()))) + .addGap(6, 6, 6) + .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(layout.createSequentialGroup() + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(postHitCheckBox))))) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -270,13 +316,50 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addComponent(newTypeButton) .addComponent(deleteTypeButton))) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(saveTypeButton)))) .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addContainerGap()) ); }// //GEN-END:initComponents + private void newTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_newTypeButtonActionPerformed + this.clearTypeDetailsComponents(); + // RJCTODO: Default name? + }//GEN-LAST:event_newTypeButtonActionPerformed + + private void deleteTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteTypeButtonActionPerformed + String typeName = this.typesList.getSelectedValue(); + this.fileTypes.remove(typeName); + this.clearTypeDetailsComponents(); + this.typesList.setSelectedIndex(-1); + }//GEN-LAST:event_deleteTypeButtonActionPerformed + + private void saveTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_saveTypeButtonActionPerformed + try { + String typeName = this.mimeTypeTextField.getText(); + if (typeName.isEmpty()) { + JOptionPane.showMessageDialog(null, "MIME type name cannot be empty.", "Missing MIME Type", JOptionPane.ERROR_MESSAGE); // RJCTODO: Bundle + return; + } + + String sigString = this.signatureTextField.getText(); + if (sigString.isEmpty()) { + JOptionPane.showMessageDialog(null, "Signature cannot be empty.", "Missing Signature", JOptionPane.ERROR_MESSAGE); // RJCTODO: Bundle + return; + } + + long offset = Long.parseUnsignedLong(this.offsetTextField.getText()); + + FileType.Signature.Type sigType = this.signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; + FileType.Signature signature = new FileType.Signature(new byte[1], offset, sigType); // RJCTODO: + FileType fileType = new FileType(typeName, signature, this.postHitCheckBox.isSelected()); + this.fileTypes.put(typeName, fileType); + } catch (NumberFormatException ex) { + JOptionPane.showMessageDialog(null, "Offset is not a positive integer.", "Invalid Offset", JOptionPane.ERROR_MESSAGE); // RJCTODO: Bundle + } + }//GEN-LAST:event_saveTypeButtonActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.ButtonGroup buttonGroup1; @@ -294,9 +377,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private javax.swing.JButton saveTypeButton; private javax.swing.JLabel signatureLabel; private javax.swing.JTextField signatureTextField; - private javax.swing.JComboBox signatureTypeComboBox; + private javax.swing.JComboBox signatureTypeComboBox; private javax.swing.JLabel signatureTypeLabel; - private javax.swing.JList typesList; + private javax.swing.JList typesList; private javax.swing.JScrollPane typesScrollPane; // End of variables declaration//GEN-END:variables diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java index f7dac840ca..c93cc9a547 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -18,8 +18,7 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; -import java.util.ArrayList; -import java.util.List; +import java.util.Map; import org.sleuthkit.datamodel.AbstractFile; /** @@ -27,16 +26,14 @@ import org.sleuthkit.datamodel.AbstractFile; */ final class UserDefinedFileTypeIdentifier { - private final List fileTypes; + private final Map fileTypes; /** * Creates an object that can do file type identification for user-defined * file types. */ UserDefinedFileTypeIdentifier() { - this.fileTypes = new ArrayList<>(); - List fileTypeDefs = UserDefinedFileTypesManager.getInstance().getFileTypes(); - this.fileTypes.addAll(fileTypeDefs); + this.fileTypes = UserDefinedFileTypesManager.getInstance().getFileTypes(); } /** @@ -48,7 +45,7 @@ final class UserDefinedFileTypeIdentifier { */ FileType identify(final AbstractFile file) { FileType type = null; - for (FileType fileType : this.fileTypes) { + for (FileType fileType : this.fileTypes.values()) { if (fileType.matches(file)) { type = fileType; break; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index c3f177a4c9..1266d3560f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -62,6 +62,13 @@ final class UserDefinedFileTypesManager { private static final String ASCII_ENCODING = "US-ASCII"; //NON-NLS private static UserDefinedFileTypesManager instance; + /** + * Predefined file types are stored in this mapping of file type names to + * file types. Access to this map is guarded by the intrinsic lock of the + * user-defined file types manager for thread-safety. + */ + private final Map predefinedFileTypes = new HashMap<>(); + /** * User-defined file types to be persisted to the user-defined file type * definitions file are stored in this mapping of file type names to file @@ -96,6 +103,10 @@ final class UserDefinedFileTypesManager { * (e.g., MIME type) and signatures. */ private UserDefinedFileTypesManager() { + /** + * Load the predefined types first so that they can be overwritten by + * any user-defined types with the same names. + */ loadPredefinedFileTypes(); loadUserDefinedFileTypes(); } @@ -109,14 +120,16 @@ final class UserDefinedFileTypesManager { /** * Create a file type that should match $MBR in Small2 image. */ - this.fileTypes.put("predefinedRAW", new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), true)); + FileType fileType = new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), true); + this.addPredefinedFileType(fileType); /** * Create a file type that should match test.txt in the Small2 image. */ // RJCTODO: Remove test type try { - this.fileTypes.put("predefinedASCII", new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true)); + fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true); + this.addPredefinedFileType(fileType); } catch (UnsupportedEncodingException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS } @@ -144,16 +157,28 @@ final class UserDefinedFileTypesManager { // catch (IndexOutOfBoundsException e) { // // do nothing // } - this.fileTypes.put("text/xml", new FileType("text/xml", new Signature(" getFileTypes() { + synchronized Map getFileTypes() { /** * It is safe to return references to the internal file type objects * because they are immutable. */ - return new ArrayList<>(this.fileTypes.values()); + return new HashMap<>(this.fileTypes); } /** - * Adds a new user-defined file type, overwriting any existing file type - * with the same type name. + * Gets the user-defined file types. * - * @param fileType The file type to add. - * @throws - * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + * @return A mapping of file type names to file types, possibly empty. */ - synchronized void addFileType(FileType fileType) throws UserDefinedFileTypesException { - this.addFileTypes(Collections.singletonList(fileType)); - } - - /** - * Adds a collection of new user-defined file types, overwriting any - * existing file types with the same type names. - * - * @param newFileTypes The file types to add. - * @throws - * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException - */ - synchronized void addFileTypes(Collection newFileTypes) throws UserDefinedFileTypesException { + synchronized Map getUserDefinedFileTypes() { /** - * It is safe to hold references to client-constructed file type objects + * It is safe to return references to the internal file type objects * because they are immutable. */ - for (FileType fileType : newFileTypes) { - this.userDefinedFileTypes.put(fileType.getTypeName(), fileType); - this.fileTypes.put(fileType.getTypeName(), fileType); - } - this.saveUserDefinedTypes(); + return new HashMap<>(this.userDefinedFileTypes); } /** - * Deletes a user-defined file type. + * Sets the user-defined file types. * - * @param fileType The file type to delete. + * @param newFileTypes A mapping of file type names to user-defined + * file types. * @throws * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException */ - synchronized void deleteFileType(FileType fileType) throws UserDefinedFileTypesException { - this.deleteFileTypes(Collections.singletonList(fileType)); - } - - /** - * Deletes a set of user-defined file types. - * - * @param deletedFileTypes The file types to delete. - * @throws - * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException - */ - synchronized void deleteFileTypes(Collection deletedFileTypes) throws UserDefinedFileTypesException { - for (FileType fileType : deletedFileTypes) { - this.userDefinedFileTypes.remove(fileType.getTypeName(), fileType); - this.fileTypes.remove(fileType.getTypeName(), fileType); - } - this.saveUserDefinedTypes(); - } - - /** - * Persists the user-defined file type definitions. - * - * @throws - * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException - */ - private void saveUserDefinedTypes() throws UserDefinedFileTypesException { - String filePath = UserDefinedFileTypesManager.getFileTypeDefinitionsFilePath(UserDefinedFileTypesManager.USER_DEFINED_TYPE_DEFINITIONS_FILE); - UserDefinedFileTypesManager.writeFileTypes(this.userDefinedFileTypes.values(), filePath); - } - - /** - * Writes a set of file type definitions to a given file. - * - * @param fileTypes The file types. - * @param filePath The destination file. - * @throws UserDefinedFileTypesException - */ - private static void writeFileTypes(Collection fileTypes, String filePath) throws UserDefinedFileTypesException { + synchronized void setUserDefinedFileTypes(Map newFileTypes) throws UserDefinedFileTypesManager.UserDefinedFileTypesException { try { - UserDefinedFileTypesManager.XMLWriter.writeFileTypes(fileTypes, filePath); + /** + * Persist the user-defined file type definitions. + */ + String filePath = UserDefinedFileTypesManager.getFileTypeDefinitionsFilePath(UserDefinedFileTypesManager.USER_DEFINED_TYPE_DEFINITIONS_FILE); + UserDefinedFileTypesManager.XMLWriter.writeFileTypes(newFileTypes.values(), filePath); + } catch (ParserConfigurationException | IOException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message } + + /** + * Clear and reinitialize the user-defined file type map. It is safe to + * hold references to file type objects obtained for the caller because + * they are immutable. + */ + this.userDefinedFileTypes.clear(); + this.userDefinedFileTypes.putAll(newFileTypes); + + /** + * Clear and reinitialize the combined file type map, loading the + * predefined types first so that they can be overwritten by any + * user-defined types with the same names. + */ + this.fileTypes.clear(); + this.fileTypes.putAll(this.predefinedFileTypes); + this.fileTypes.putAll(this.userDefinedFileTypes); } /** From c7b577169288550626c335785f7b21ae420a9322 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Thu, 4 Dec 2014 22:40:08 -0800 Subject: [PATCH 32/84] Translation complete. --- Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties index edf52bb547..480522e021 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/actions/Bundle_ja.properties @@ -45,4 +45,7 @@ GetTagNameDialog.dupTagErr=\u91CD\u8907\u30BF\u30B0\u306E\u30A8\u30E9\u30FC AddContentTagAction.cannotApplyTagErr=\u30BF\u30B0\u3092\u9069\u7528\u3067\u304D\u307E\u305B\u3093 OpenLogFolder.error1=\u30ED\u30B0\u30D5\u30A1\u30A4\u30EB\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F\uFF1A{0} CTL_OpenLogFolder=\u30ED\u30B0\u30D5\u30A9\u30EB\u30C0\u3092\u958B\u304F -ShowIngestProgressSnapshotAction.actionName.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30D7\u30ED\u30B0\u30EC\u30B9\u306E\u30B9\u30CA\u30C3\u30D7\u30B7\u30E7\u30C3\u30C8\u3092\u53D6\u5F97 \ No newline at end of file +ShowIngestProgressSnapshotAction.actionName.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30D7\u30ED\u30B0\u30EC\u30B9\u306E\u30B9\u30CA\u30C3\u30D7\u30B7\u30E7\u30C3\u30C8\u3092\u53D6\u5F97 +CTL_OpenPythonModulesFolderAction=Python\u30D7\u30E9\u30B0\u30A4\u30F3 +OpenPythonModulesFolderAction.actionName.text=Python\u30D7\u30E9\u30B0\u30A4\u30F3 +OpenPythonModulesFolderAction.errorMsg.folderNotFound=Python\u30D7\u30E9\u30B0\u30A4\u30F3\u30D5\u30A9\u30EB\u30C0\u30FC\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F\uFF1A{0} \ No newline at end of file From 93ee7f2ae0e159de6d8acaf456aee64bbe34f3f6 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Fri, 5 Dec 2014 11:46:03 -0800 Subject: [PATCH 33/84] Translation complete. --- .../autopsy/modules/interestingitems/Bundle_ja.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index 30b5208be7..b336ecfd14 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -41,7 +41,7 @@ InterestingItemDefsPanel.jLabel3.text=\u30CD\u30FC\u30E0\u30D1\u30BF\u30FC\u30F3 InterestingItemDefsPanel.jLabel4.text=\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\uFF1A InterestingItemDefsPanel.jLabel5.text=\u6982\u8981\uFF1A InterestingItemDefsPanel.jLabel6.text=\u30BB\u30C3\u30C8\u8A73\u7D30 -InterestingItemDefsPanel.jTextArea1.text=\u6307\u5B9A\u3055\u308C\u305F\u6761\u4EF6\u3068\u4E00\u81F4\u3059\u308B\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u7D22\u3059\u308B\u306E\u304C\u53EF\u80FD\u306A\u30E2\u30B8\u30E5\u30FC\u30EB\u3067\u3059\u3002\u5404\u30BB\u30C3\u30C8\u306B\u306F\u30D5\u30A1\u30A4\u30EB\u540D\u304A\u3088\u3073\u30DA\u30A2\u30EC\u30F3\u30C8\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\u3092\u3082\u3068\u306B\u4E00\u81F4\u3059\u308B\u30EB\u30FC\u30EB\u30EA\u30B9\u30C8\u304C\u3042\u308A\u307E\u3059\u3002 +InterestingItemDefsPanel.jTextArea1.text=\u6307\u5B9A\u3055\u308C\u305F\u6761\u4EF6\u3068\u4E00\u81F4\u3059\u308B\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u7D22\u3059\u308B\u306E\u304C\u53EF\u80FD\u306A\u30E2\u30B8\u30E5\u30FC\u30EB\u3067\u3059\u3002\u5404\u30BB\u30C3\u30C8\u306B\u306F\u30D5\u30A1\u30A4\u30EB\u540D\u304A\u3088\u3073\u30DA\u30A2\u30EC\u30F3\u30C8\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\u3092\u3082\u3068\u306B\u4E00\u81F4\u3059\u308B\u30EB\u30FC\u30EB\u306E\u30EA\u30B9\u30C8\u304C\u3042\u308A\u307E\u3059\u3002 InterestingItemDefsPanel.newRuleButton.text=\u65B0\u898F\u30EB\u30FC\u30EB InterestingItemDefsPanel.newSetButton.text=\u65B0\u898F\u30BB\u30C3\u30C8 InterestingItemDefsPanel.rulePathFilterRegexCheckBox.text=\u6B63\u898F\u8868\u73FE @@ -50,6 +50,6 @@ InterestingItemDefsPanel.setsListLabel.text=\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8 InterestingItemsIdentifierIngestModule.moduleDescription=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u3092\u898B\u3064\u3051\u307E\u3059\u3002 InterestingItemsIdentifierIngestModule.moduleName=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB -OpenIDE-Module-Long-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u3092\u898B\u3064\u3051\u307E\u3059 +OpenIDE-Module-Long-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u77E5\u3057\u307E\u3059\u3002 OpenIDE-Module-Name=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 OpenIDE-Module-Short-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB \ No newline at end of file From 9c94d07dc17175acac3525e7ca74af821c80fac6 Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Fri, 5 Dec 2014 11:50:17 -0800 Subject: [PATCH 34/84] Translation complete. --- .../autopsy/modules/interestingitems/Bundle_ja.properties | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index b336ecfd14..7660216076 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -52,4 +52,6 @@ InterestingItemsIdentifierIngestModule.moduleName=\u7591\u308F\u3057\u3044\u30D5 OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB OpenIDE-Module-Long-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u77E5\u3057\u307E\u3059\u3002 OpenIDE-Module-Name=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 -OpenIDE-Module-Short-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB \ No newline at end of file +OpenIDE-Module-Short-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB +OptionsCategory_Keywords_InterestingItemDefinitions=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u5B9A\u7FA9 +OptionsCategory_Name_InterestingItemDefinitions=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u5B9A\u7FA9 \ No newline at end of file From 401c725f67bc838c985fc642803c5ed387a7467a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 5 Dec 2014 14:54:18 -0500 Subject: [PATCH 35/84] Additional work on user defined types --- .../modules/filetypeid/Bundle.properties | 10 ++ .../FileTypeIdGlobalSettingsPanel.form | 6 +- .../FileTypeIdGlobalSettingsPanel.java | 141 ++++++++++++++---- .../FileTypeIdOptionsPanelController.java | 9 +- .../UserDefinedFileTypesManager.java | 1 - 5 files changed, 123 insertions(+), 44 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 2d18364283..6a15e196d1 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -21,3 +21,13 @@ FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text= FileTypeIdGlobalSettingsPanel.signatureLabel.text=Signature FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text=Mime Type FileTypeIdGlobalSettingsPanel.saveTypeButton.text=Save Type +FileTypeIdGlobalSettingsPanel.signatureComboBox.rawItem=Bytes (Hex) +FileTypeIdGlobalSettingsPanel.signatureComboBox.asciiItem=String (ASCII) +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.message=MIME type is required. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.title=Missing MIME Type +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message=Signature is required. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title=Missing Signature +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message=Offset must be a positive integer. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title=Invalid Offset +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message=The signature must be able to be converted to UTF-8 bytes. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Signature \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index 49ae296a57..3d5852a878 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -1,10 +1,6 @@
    - - - - @@ -46,7 +42,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index aaafa786eb..169c69ddc7 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -18,12 +18,17 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import javax.swing.DefaultComboBoxModel; import javax.swing.DefaultListModel; import javax.swing.JOptionPane; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; +import javax.xml.bind.DatatypeConverter; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; @@ -34,20 +39,54 @@ import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; */ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { - private static final String RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM = "Bytes (Hex)"; // RJCTODO: Bundle - private static final String ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM = "String (ASCII)"; // RJCTODO: Bundle + private static final String RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM = NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureComboBox.rawItem"); + private static final String ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM = NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureComboBox.asciiItem"); + + /** + * These two fields are used to synthesize default names for user-defined + * types. This is a thread-safe implementation because there can be two + * instances of this panel at the same time due to the non-modal nature of + * the NetBeans options window and the fact that these panels can also be + * invoked via ingest job configuration panels. All interactions with + * instances of this panel should occur on the EDT, so this is defensive + * programming. + */ + private static final String DEFAULT_TYPE_NAME_BASE = "userdefined/userdefined"; //NON-NLS + private static final AtomicInteger defaultTypeNameCounter = new AtomicInteger(1); /** * This mapping of file type names to file types is used to hold the types - * displayed in the file types list component. + * displayed in the file types list component via its list model. */ private Map fileTypes; + private DefaultListModel typesListModel; /** * Creates a panel to allow a user to make custom file type definitions. */ FileTypeIdGlobalSettingsPanel() { - initComponents(); + this.initComponents(); + this.customizeComponents(); + } + + /** + * Does component initialization in addition to the the Matisse generated + * initialization. + */ + private void customizeComponents() { + /** + * Make a model for the file types list component. + */ + this.typesListModel = new DefaultListModel<>(); + this.typesList.setModel(this.typesListModel); + + /** + * Make a model for the signature type combo box component. + */ + DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); } /** @@ -68,11 +107,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * types list component. */ this.fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); - DefaultListModel fileTypesListModel = new DefaultListModel<>(); - for (FileType fileType : this.fileTypes.values()) { - fileTypesListModel.addElement(fileType.getTypeName()); - } - this.typesList.setModel(fileTypesListModel); + this.setFileTypesListModel(); /** * Add a selection listener to populate the file type details @@ -84,17 +119,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * If there is at least one user-defined file type, select it the file * types list component. */ - if (!fileTypesListModel.isEmpty()) { + if (!this.typesListModel.isEmpty()) { this.typesList.setSelectedIndex(0); } - - /** - * Make a model for the signature type combo box component. - */ - DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); - sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); - sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); - this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); } /** @@ -109,6 +136,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } } + /** + * Selection listener for the file types list component. + */ private class TypesListSelectionListener implements ListSelectionListener { @Override @@ -132,18 +162,30 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } /** - * RJCTODO + * Sets the list model for the file types list component. + */ + private void setFileTypesListModel() { + ArrayList typeNames = new ArrayList(this.fileTypes.keySet()); + Collections.sort(typeNames); + this.typesListModel.clear(); + for (String typeName : typeNames) { + this.typesListModel.addElement(typeName); + } + } + + /** + * Clears all of the components in the individual type details portion of + * the panel. */ private void clearTypeDetailsComponents() { this.typesList.setSelectedIndex(-1); - this.mimeTypeTextField.setText(""); //NON-NLS // RJCTODO: Default name? + this.mimeTypeTextField.setText(""); this.signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); this.signatureTextField.setText(""); //NON-NLS this.offsetTextField.setText(""); //NON-NLS this.postHitCheckBox.setSelected(false); } - // RJCTODO: Consider fixing OptionsPanel interface or writing story /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -153,7 +195,6 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane // //GEN-BEGIN:initComponents private void initComponents() { - buttonGroup1 = new javax.swing.ButtonGroup(); typesScrollPane = new javax.swing.JScrollPane(); typesList = new javax.swing.JList(); jSeparator1 = new javax.swing.JSeparator(); @@ -252,7 +293,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() .addComponent(signatureTypeLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 82, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(signatureTypeComboBox, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() @@ -325,7 +366,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private void newTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_newTypeButtonActionPerformed this.clearTypeDetailsComponents(); - // RJCTODO: Default name? + this.mimeTypeTextField.setText(FileTypeIdGlobalSettingsPanel.DEFAULT_TYPE_NAME_BASE + FileTypeIdGlobalSettingsPanel.defaultTypeNameCounter.getAndIncrement()); }//GEN-LAST:event_newTypeButtonActionPerformed private void deleteTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteTypeButtonActionPerformed @@ -337,32 +378,70 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private void saveTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_saveTypeButtonActionPerformed try { + /** + * Get the file type name. + */ String typeName = this.mimeTypeTextField.getText(); if (typeName.isEmpty()) { - JOptionPane.showMessageDialog(null, "MIME type name cannot be empty.", "Missing MIME Type", JOptionPane.ERROR_MESSAGE); // RJCTODO: Bundle + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.title"), + JOptionPane.ERROR_MESSAGE); return; } + /** + * Get the signature type. + */ + FileType.Signature.Type sigType = this.signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; + + /** + * Get the signature bytes. + */ String sigString = this.signatureTextField.getText(); if (sigString.isEmpty()) { - JOptionPane.showMessageDialog(null, "Signature cannot be empty.", "Missing Signature", JOptionPane.ERROR_MESSAGE); // RJCTODO: Bundle + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title"), + JOptionPane.ERROR_MESSAGE); return; } + byte[] signatureBytes; + if (FileType.Signature.Type.RAW == sigType) { + signatureBytes = DatatypeConverter.parseHexBinary(sigString); + } else { + signatureBytes = sigString.getBytes(Charset.forName("UTF-8")); + } + /** + * Get the offset. + */ long offset = Long.parseUnsignedLong(this.offsetTextField.getText()); - - FileType.Signature.Type sigType = this.signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; - FileType.Signature signature = new FileType.Signature(new byte[1], offset, sigType); // RJCTODO: + + /** + * Put it all together and reset the file types list component. + */ + FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); // RJCTODO: FileType fileType = new FileType(typeName, signature, this.postHitCheckBox.isSelected()); this.fileTypes.put(typeName, fileType); + this.setFileTypesListModel(); + this.typesList.setSelectedValue(fileType.getTypeName(), true); + } catch (NumberFormatException ex) { - JOptionPane.showMessageDialog(null, "Offset is not a positive integer.", "Invalid Offset", JOptionPane.ERROR_MESSAGE); // RJCTODO: Bundle + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), + JOptionPane.ERROR_MESSAGE); + } catch (IllegalArgumentException ex) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title"), + JOptionPane.ERROR_MESSAGE); } }//GEN-LAST:event_saveTypeButtonActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.ButtonGroup buttonGroup1; private javax.swing.JButton deleteTypeButton; private javax.swing.JLabel hexPrefixLabel; private javax.swing.JScrollPane jScrollPane2; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java index 45d24b9db4..bbf16073de 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java @@ -34,13 +34,8 @@ public final class FileTypeIdOptionsPanelController extends OptionsPanelControll @Override public void applyChanges() { - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - getPanel().store(); - changed = false; - } - }); + getPanel().store(); + changed = false; } @Override diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 1266d3560f..031a6d46f8 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -25,7 +25,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; From 8f93988d259fc23e21fcb528f8cc3a4e45a20ad7 Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Fri, 5 Dec 2014 16:26:29 -0500 Subject: [PATCH 36/84] Removed translation for Module Keyword. --- .../autopsy/modules/interestingitems/Bundle_ja.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index 7660216076..1aaccb3c4f 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -53,5 +53,4 @@ OpenIDE-Module-Display-Category=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8 OpenIDE-Module-Long-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3002\n\n\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30EB\u30FC\u30EB\u30BB\u30C3\u30C8\u306E\u5B9A\u7FA9\u3092\u3082\u3068\u306B\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u77E5\u3057\u307E\u3059\u3002 OpenIDE-Module-Name=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2 OpenIDE-Module-Short-Description=\u7591\u308F\u3057\u3044\u30D5\u30A1\u30A4\u30EB\u30A2\u30A4\u30C7\u30F3\u30C6\u30A3\u30D5\u30A1\u30A4\u30A2\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB -OptionsCategory_Keywords_InterestingItemDefinitions=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u5B9A\u7FA9 OptionsCategory_Name_InterestingItemDefinitions=\u7591\u308F\u3057\u3044\u30A2\u30A4\u30C6\u30E0\u5B9A\u7FA9 \ No newline at end of file From 6e9c22032bb377041e794803f1618fbf05a6e829 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sun, 7 Dec 2014 08:46:39 -0500 Subject: [PATCH 37/84] Additions to user-defined types feature --- .../autopsy/modules/filetypeid/FileType.java | 2 +- .../FileTypeIdGlobalSettingsPanel.form | 23 +++------ .../FileTypeIdGlobalSettingsPanel.java | 49 ++++++++++--------- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java index 98b174b46f..8aa261a84f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -110,7 +110,7 @@ final class FileType { * Creates a file signature consisting of a sequence of bytes at a * specific offset within a file. * - * @param signatureBytes The signature bytes + * @param signatureBytes The signature bytes. * @param offset The offset of the signature bytes. * @param type The interpretation of the signature bytes (e.g., raw * bytes, an ASCII string). diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index 3d5852a878..a2dbc9c05d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -50,14 +50,10 @@ - - - - - + - + @@ -121,18 +117,13 @@ + - - - - - - - - - - + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 169c69ddc7..3fdb28898e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -35,7 +35,9 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; /** - * A panel to allow a user to make custom file type definitions. + * A panel to allow a user to make custom file type definitions. In addition to + * being an ingest module global settings panel, an instance of this class also + * appears in the NetBeans options dialog as an options panel. */ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { @@ -44,10 +46,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * These two fields are used to synthesize default names for user-defined - * types. This is a thread-safe implementation because there can be two - * instances of this panel at the same time due to the non-modal nature of - * the NetBeans options window and the fact that these panels can also be - * invoked via ingest job configuration panels. All interactions with + * types. This is a thread-safe implementation. All interactions with * instances of this panel should occur on the EDT, so this is defensive * programming. */ @@ -55,11 +54,12 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private static final AtomicInteger defaultTypeNameCounter = new AtomicInteger(1); /** - * This mapping of file type names to file types is used to hold the types - * displayed in the file types list component via its list model. + * The list model for the file types list component of this panel is the set + * of type names of the user-defined file types. A mapping of the file type + * names to file type objects completes the model. */ - private Map fileTypes; private DefaultListModel typesListModel; + private Map fileTypes; /** * Creates a panel to allow a user to make custom file type definitions. @@ -70,8 +70,8 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } /** - * Does component initialization in addition to the the Matisse generated - * initialization. + * Does child component initialization in addition to the the Matisse + * generated initialization. */ private void customizeComponents() { /** @@ -79,14 +79,14 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane */ this.typesListModel = new DefaultListModel<>(); this.typesList.setModel(this.typesListModel); - + /** * Make a model for the signature type combo box component. */ DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); - this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); + this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); } /** @@ -98,7 +98,8 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } /** - * Populates the child components. + * Populates the child components with file types obtained from the + * user-defined file types manager. */ @Override public void load() { @@ -413,11 +414,11 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane signatureBytes = sigString.getBytes(Charset.forName("UTF-8")); } - /** + /** * Get the offset. */ long offset = Long.parseUnsignedLong(this.offsetTextField.getText()); - + /** * Put it all together and reset the file types list component. */ @@ -426,17 +427,17 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane this.fileTypes.put(typeName, fileType); this.setFileTypesListModel(); this.typesList.setSelectedValue(fileType.getTypeName(), true); - + } catch (NumberFormatException ex) { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), - JOptionPane.ERROR_MESSAGE); + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), + JOptionPane.ERROR_MESSAGE); } catch (IllegalArgumentException ex) { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title"), - JOptionPane.ERROR_MESSAGE); + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title"), + JOptionPane.ERROR_MESSAGE); } }//GEN-LAST:event_saveTypeButtonActionPerformed From c413aae3771eaa27bde7e92e034ac6a3325c4ef6 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 8 Dec 2014 09:05:42 -0500 Subject: [PATCH 38/84] Continue to work on user-defined types --- .../modules/filetypeid/FileTypeIdGlobalSettingsPanel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 3fdb28898e..32b4aff441 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -51,7 +51,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * programming. */ private static final String DEFAULT_TYPE_NAME_BASE = "userdefined/userdefined"; //NON-NLS - private static final AtomicInteger defaultTypeNameCounter = new AtomicInteger(1); + private static final AtomicInteger defaultTypeNameCounter = new AtomicInteger(1); // RJCTODO: Need to save and init counter /** * The list model for the file types list component of this panel is the set From e5124eed28b730a92559dc07fba607ca701866b8 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 8 Dec 2014 09:17:21 -0500 Subject: [PATCH 39/84] Correct javadocs for FileINgestModule.shutdown() --- .../org/sleuthkit/autopsy/ingest/FileIngestModule.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java index 369e34925c..42dead516c 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java @@ -38,10 +38,10 @@ public interface FileIngestModule extends IngestModule { /** * Invoked by Autopsy when an ingest job is completed (either because the * data has been analyzed or because the job was canceled - check - * IngestJobContext.isJobCancelled()), before the ingest module instance is - * discarded. The module should respond by doing things like releasing - * private resources, submitting final results, and posting a final ingest - * message. + * IngestJobContext.fileIngestIsCancelled()), before the ingest module + * instance is discarded. The module should respond by doing things like + * releasing private resources, submitting final results, and posting a + * final ingest message. */ void shutDown(); } From 552d1039fb2e55713b3c99d8be9d67e7f2c1759b Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 8 Dec 2014 19:30:26 -0500 Subject: [PATCH 40/84] Improve user-defined file types (incomplete) --- .../sleuthkit/autopsy/coreutils/XMLUtil.java | 48 +++++ .../modules/filetypeid/Bundle.properties | 6 +- .../autopsy/modules/filetypeid/FileType.java | 25 ++- .../FileTypeIdGlobalSettingsPanel.form | 181 ++++++++--------- .../FileTypeIdGlobalSettingsPanel.java | 182 ++++++++++-------- .../filetypeid/UserDefinedFileTypes.xsd | 11 +- .../UserDefinedFileTypesManager.java | 126 +++++++----- 7 files changed, 350 insertions(+), 229 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java index dc12e3bd2c..603fc52e60 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java @@ -25,6 +25,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; +import java.nio.file.Paths; import java.util.logging.Level; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; @@ -155,6 +156,7 @@ public class XMLUtil { * @param xmlPath the full path to the file to load * @param xsdPath the full path to the file to validate against */ + // RJCTODO: Deprecate public static Document loadDoc(Class clazz, String xmlPath, String xsdPath) { Document ret = loadDoc(clazz, xmlPath); if (!XMLUtil.xmlIsValid(ret, clazz, xsdPath)) { @@ -163,6 +165,51 @@ public class XMLUtil { return ret; } + /** + * Used to consolidate more specific exception types. + */ + public static class XmlUtilException extends Exception { + + XmlUtilException(String message) { + super(message); + } + } + + /** + * Loads and validates an XML document. + * + * @param docPath The full path to the file to load. + * @param schemaPath The full path to the file to validate against. + */ + /** + * Loads and XML document and validates against a schema packaged as a + * class resource. + * + * @param The name of the class associated with the resource. + * @param clazz The class associated with the resource. + * @param docPath The full path to the XML document. + * @param schemaResourceName The name of the schema resource + * @return A WC3 DOM representation of the document + * @throws IOException + * @throws org.sleuthkit.autopsy.coreutils.XMLUtil.XmlUtilException + */ + public static Document loadAndValidateDoc(Class clazz, String docPath, String schemaResourceName) throws IOException, XmlUtilException { + try { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document doc = builder.parse(new FileInputStream(docPath)); + PlatformUtil.extractResourceToUserConfigDir(clazz, schemaResourceName, false); + File schemaFile = new File(Paths.get(PlatformUtil.getUserConfigDirectory(), schemaResourceName).toAbsolutePath().toString()); + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = schemaFactory.newSchema(schemaFile); + Validator validator = schema.newValidator(); + validator.validate(new DOMSource(doc), new DOMResult()); + return doc; + } catch (ParserConfigurationException | SAXException ex) { + throw new XmlUtilException(ex.getLocalizedMessage()); + } + } + /** * Saves XML files to disk * @@ -203,4 +250,5 @@ public class XMLUtil { } return success; } + } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 6a15e196d1..2f55fc8fb8 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -19,7 +19,7 @@ FileTypeIdGlobalSettingsPanel.jTextArea1.text=Enter a MIME type and signature to FileTypeIdGlobalSettingsPanel.signatureTypeLabel.text=Signature Type FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text= FileTypeIdGlobalSettingsPanel.signatureLabel.text=Signature -FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text=Mime Type +FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text=MIME Type FileTypeIdGlobalSettingsPanel.saveTypeButton.text=Save Type FileTypeIdGlobalSettingsPanel.signatureComboBox.rawItem=Bytes (Hex) FileTypeIdGlobalSettingsPanel.signatureComboBox.asciiItem=String (ASCII) @@ -30,4 +30,6 @@ FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title=Missing Signatu FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message=Offset must be a positive integer. FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title=Invalid Offset FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message=The signature must be able to be converted to UTF-8 bytes. -FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Signature \ No newline at end of file +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Signature +FileTypeIdGlobalSettingsPanel.filesSetNameLabel.text=Files Set Name +FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text= diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java index 8aa261a84f..e3b9908131 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -29,32 +29,35 @@ import org.sleuthkit.datamodel.TskCoreException; */ final class FileType { - private final String typeName; + private final String mimeType; private final Signature signature; + private final String filesSetName; private final boolean alert; /** * Creates a representation of a named file type characterized by a file * signature. * - * @param typeName The name of the file type. + * @param mimeType The mime type to associate with this file type. * @param signature The signature that characterizes the file type. + * @param filesSetName The interesting files set name * @param alert A flag indicating whether the user wishes to be alerted when * a file matching this type is encountered. */ - FileType(String typeName, final Signature signature, boolean alert) { - this.typeName = typeName; + FileType(String mimeType, final Signature signature, String filesSetName, boolean alert) { + this.mimeType = mimeType; this.signature = new Signature(signature.getSignatureBytes(), signature.getOffset(), signature.getType()); + this.filesSetName = filesSetName; this.alert = alert; } /** - * Gets the name associated with this file type. + * Gets the MIME type associated with this file type. * * @return The type name. */ - String getTypeName() { - return this.typeName; + String getMimeType() { + return this.mimeType; } /** @@ -85,6 +88,14 @@ final class FileType { boolean alertOnMatch() { return this.alert; } + + /** + * Gets the interesting files set name assigned to this file type + * @return + */ + String getFilesSetName() { + return this.filesSetName; + } /** * A file signature consisting of a sequence of bytes at a specific offset diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index a2dbc9c05d..a4615bbc52 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -20,115 +20,107 @@ - + + + + + + - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + + + + + - + - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + @@ -284,6 +276,23 @@ + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 32b4aff441..70aebae4d5 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -87,6 +87,8 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); + + this.filesSetNameTextField.setEnabled(false); } /** @@ -214,6 +216,8 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane jScrollPane2 = new javax.swing.JScrollPane(); jTextArea1 = new javax.swing.JTextArea(); postHitCheckBox = new javax.swing.JCheckBox(); + filesSetNameLabel = new javax.swing.JLabel(); + filesSetNameTextField = new javax.swing.JTextField(); typesScrollPane.setViewportView(typesList); @@ -266,6 +270,15 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane jScrollPane2.setViewportView(jTextArea1); org.openide.awt.Mnemonics.setLocalizedText(postHitCheckBox, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.postHitCheckBox.text")); // NOI18N + postHitCheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + postHitCheckBoxActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(filesSetNameLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.filesSetNameLabel.text")); // NOI18N + + filesSetNameTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text")); // NOI18N javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -275,93 +288,85 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addGap(26, 26, 26) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() - .addGap(10, 10, 10) + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addGap(0, 0, Short.MAX_VALUE) .addComponent(newTypeButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(deleteTypeButton)) - .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)) + .addComponent(deleteTypeButton) + .addGap(9, 9, 9))) + .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addGap(37, 37, 37) - .addComponent(saveTypeButton)) - .addGroup(layout.createSequentialGroup() - .addGap(18, 18, 18) - .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) - .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() - .addComponent(signatureTypeLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(signatureTypeComboBox, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) - .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addComponent(signatureLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(hexPrefixLabel) - .addGap(2, 2, 2)) - .addGroup(layout.createSequentialGroup() - .addComponent(offsetLabel) - .addGap(44, 44, 44))) - .addGap(5, 5, 5) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 84, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() - .addGap(2, 2, 2) - .addComponent(mimeTypeLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 181, javax.swing.GroupLayout.PREFERRED_SIZE)))) - .addGroup(layout.createSequentialGroup() - .addGap(6, 6, 6) - .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGroup(layout.createSequentialGroup() - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(postHitCheckBox))))) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addComponent(signatureTypeLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(signatureTypeComboBox, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(offsetLabel) + .addComponent(filesSetNameLabel) + .addGroup(layout.createSequentialGroup() + .addComponent(signatureLabel) + .addGap(18, 18, 18) + .addComponent(hexPrefixLabel))) + .addGap(5, 5, 5) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 84, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGap(2, 2, 2) + .addComponent(mimeTypeLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 181, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 179, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(postHitCheckBox) + .addComponent(saveTypeButton)) + .addContainerGap(29, Short.MAX_VALUE)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(16, 16, 16) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) + .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 281, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() - .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(18, 18, 18) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(mimeTypeLabel) - .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(signatureTypeLabel) - .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(hexPrefixLabel) - .addComponent(signatureLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(offsetLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(postHitCheckBox))) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) - .addGroup(layout.createSequentialGroup() - .addGap(18, 18, 18) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(newTypeButton) - .addComponent(deleteTypeButton))) - .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(saveTypeButton)))) - .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addContainerGap()) + .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGap(18, 18, 18) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(mimeTypeLabel) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(signatureTypeLabel) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(hexPrefixLabel) + .addComponent(signatureLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(offsetLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(postHitCheckBox) + .addGap(8, 8, 8) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(filesSetNameLabel) + .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(saveTypeButton)) + .addGroup(layout.createSequentialGroup() + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 249, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(deleteTypeButton) + .addComponent(newTypeButton)))) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); }// //GEN-END:initComponents @@ -419,14 +424,25 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane */ long offset = Long.parseUnsignedLong(this.offsetTextField.getText()); + /** + * Get the interesting files set details. + */ + String filesSetName = this.filesSetNameTextField.getText(); + if (this.postHitCheckBox.isSelected() && filesSetName.isEmpty()) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), + JOptionPane.ERROR_MESSAGE); + } + /** * Put it all together and reset the file types list component. */ FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); // RJCTODO: - FileType fileType = new FileType(typeName, signature, this.postHitCheckBox.isSelected()); + FileType fileType = new FileType(typeName, signature, filesSetName, this.postHitCheckBox.isSelected()); this.fileTypes.put(typeName, fileType); this.setFileTypesListModel(); - this.typesList.setSelectedValue(fileType.getTypeName(), true); + this.typesList.setSelectedValue(fileType.getMimeType(), true); } catch (NumberFormatException ex) { JOptionPane.showMessageDialog(null, @@ -441,9 +457,15 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } }//GEN-LAST:event_saveTypeButtonActionPerformed + private void postHitCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_postHitCheckBoxActionPerformed + this.filesSetNameTextField.setEnabled(this.postHitCheckBox.isSelected()); + }//GEN-LAST:event_postHitCheckBoxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton deleteTypeButton; + private javax.swing.JLabel filesSetNameLabel; + private javax.swing.JTextField filesSetNameTextField; private javax.swing.JLabel hexPrefixLabel; private javax.swing.JScrollPane jScrollPane2; private javax.swing.JSeparator jSeparator1; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd index aa8e053534..16557dc422 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd @@ -30,11 +30,18 @@ + + + + + + + - + - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 031a6d46f8..53ec537fa6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -41,8 +41,8 @@ import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; /** - * Manages user-defined file types characterized by type names (e.g., MIME type) - * and signatures. + * Manages user-defined file types characterized by MIME type, signature, and + * optional membership in an interesting files set. */ final class UserDefinedFileTypesManager { @@ -51,19 +51,20 @@ final class UserDefinedFileTypesManager { private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; //NON-NLS private static final String FILE_TYPES_TAG_NAME = "filetypes"; //NON-NLS private static final String FILE_TYPE_TAG_NAME = "filetype"; //NON-NLS - private static final String ALERT_ATTRIBUTE = "alert"; //NON-NLS - private static final String TYPE_NAME_TAG_NAME = "typename"; //NON-NLS + private static final String MIME_TYPE_TAG_NAME = "mimetype"; //NON-NLS private static final String SIGNATURE_TAG_NAME = "signature"; //NON-NLS private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; //NON-NLS private static final String BYTES_TAG_NAME = "bytes"; //NON-NLS private static final String OFFSET_TAG_NAME = "offset"; //NON-NLS + private static final String INTERESTING_FILES_SET_TAG_NAME = "filesset"; //NON-NLS + private static final String ALERT_ATTRIBUTE = "alert"; //NON-NLS private static final String ENCODING_FOR_XML_FILE = "UTF-8"; //NON-NLS private static final String ASCII_ENCODING = "US-ASCII"; //NON-NLS private static UserDefinedFileTypesManager instance; /** - * Predefined file types are stored in this mapping of file type names to - * file types. Access to this map is guarded by the intrinsic lock of the + * Predefined file types are stored in this mapping of MIME types to file + * types. Access to this map is guarded by the intrinsic lock of the * user-defined file types manager for thread-safety. */ private final Map predefinedFileTypes = new HashMap<>(); @@ -78,17 +79,18 @@ final class UserDefinedFileTypesManager { /** * The combined set of user-defined file types and file types predefined by - * Autopsy are stored in this mapping of file type names to file types. This - * is the current working set of file types. Access to this map is guarded - * by the intrinsic lock of the user-defined file types manager for + * Autopsy are stored in this mapping of MIME types to file types. This is + * the current working set of file types. Access to this map is guarded by + * the intrinsic lock of the user-defined file types manager for * thread-safety. */ private final Map fileTypes = new HashMap<>(); /** - * Gets the user-defined file types manager. + * Gets the manager of user-defined file types characterized by MIME type, + * signature, and optional membership in an interesting files set. * - * @return A singleton user-defined file types manager. + * @return The user-defined file types manager singleton. */ synchronized static UserDefinedFileTypesManager getInstance() { if (UserDefinedFileTypesManager.instance == null) { @@ -98,8 +100,8 @@ final class UserDefinedFileTypesManager { } /** - * Creates a manager of user-defined file types characterized by type names - * (e.g., MIME type) and signatures. + * Creates a manager of user-defined file types characterized by MIME type, + * signature, and optional membership in an interesting files set. */ private UserDefinedFileTypesManager() { /** @@ -111,30 +113,30 @@ final class UserDefinedFileTypesManager { } /** - * Adds standard predefined file types to the in-memory mapping of file type - * names to predefined file types. + * Adds the predefined file types to the in-memory mappings of MIME types to + * file types. */ private void loadPredefinedFileTypes() { - // RJCTODO: Remove test type + // RJCTODO: Remove test file type. /** * Create a file type that should match $MBR in Small2 image. */ - FileType fileType = new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), true); + FileType fileType = new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), "predefinedRAW", true); this.addPredefinedFileType(fileType); /** * Create a file type that should match test.txt in the Small2 image. */ - // RJCTODO: Remove test type + // RJCTODO: Remove test file type. try { - fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), true); + fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), "predefinedASCII", true); this.addPredefinedFileType(fileType); } catch (UnsupportedEncodingException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS } try { - // RJCTODO: Remove this code from TikaFileTypeDetector.java + // RJCTODO: Remove this code from TikaFileTypeDetector.java. // try { // byte buf[]; // int len = abstractFile.read(buffer, 0, BUFFER_SIZE); @@ -156,7 +158,7 @@ final class UserDefinedFileTypesManager { // catch (IndexOutOfBoundsException e) { // // do nothing // } - fileType = new FileType("text/xml", new Signature(" Date: Mon, 8 Dec 2014 23:10:21 -0500 Subject: [PATCH 41/84] Restore batching of data sources to ingest jobs start API --- .../AddImageWizardIngestConfigPanel.java | 5 +- .../autopsy/ingest/Bundle.properties | 1 + .../autopsy/ingest/IngestJobConfigurator.java | 13 +- .../autopsy/ingest/IngestManager.java | 188 ++++++++++-------- .../ingest/RunIngestModulesDialog.java | 7 +- 5 files changed, 114 insertions(+), 100 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 268404faa0..96208f007f 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,10 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel dataSources) { - IngestManager ingestManager = IngestManager.getInstance(); - for (Content dataSource : dataSources) { - ingestManager.startIngestJob(dataSource, this.settings, true); - } + IngestManager.getInstance().startIngestJobs(dataSources, this.settings, true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3a9a58a959..e2f553ca3c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -22,6 +22,7 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -65,7 +66,7 @@ public class IngestManager { private final ExecutorService fileIngestThreadPool; private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); private final AtomicLong nextThreadId = new AtomicLong(0L); - private final ConcurrentHashMap> startIngestJobsCallables = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. + private final ConcurrentHashMap> ingestJobStarters = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); @@ -156,15 +157,14 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for a data - * source. + * Starts an ingest job, i.e., processing by ingest modules, for a + * collection of data sources. * - * @param dataSource The data source to be processed. + * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param doStartupErrorsMsgBox Whether or not to display ingest module - * startup errors in a message box. + * @param doMessageBoxes Whether or not to display message boxes for errors. */ - public synchronized void startIngestJob(Content dataSource, IngestJobSettings settings, boolean doStartupErrorsMsgBox) { + public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doMessageBoxes) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -174,8 +174,8 @@ public class IngestManager { } long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new StartIngestJobsCallable(taskId, dataSource, settings, doStartupErrorsMsgBox)); - startIngestJobsCallables.put(taskId, task); + Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, doMessageBoxes)); + ingestJobStarters.put(taskId, task); } /** @@ -192,7 +192,7 @@ public class IngestManager { */ public void cancelAllIngestJobs() { // Stop creating new ingest jobs. - for (Future handle : startIngestJobsCallables.values()) { + for (Future handle : ingestJobStarters.values()) { handle.cancel(true); } @@ -297,7 +297,7 @@ public class IngestManager { */ private void startDataSourceIngestThread() { long threadId = nextThreadId.incrementAndGet(); - dataSourceIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); + dataSourceIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } @@ -307,7 +307,7 @@ public class IngestManager { */ private void startFileIngestThread() { long threadId = nextThreadId.incrementAndGet(); - fileIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); + fileIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } @@ -442,7 +442,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobStarted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); } /** @@ -451,7 +451,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCompleted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); } /** @@ -460,7 +460,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCancelled(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); } /** @@ -469,7 +469,7 @@ public class IngestManager { * @param file The file that is completed. */ void fireFileIngestDone(AbstractFile file) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); } /** @@ -478,7 +478,7 @@ public class IngestManager { * @param moduleDataEvent A ModuleDataEvent with the details of the posting. */ void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); } /** @@ -489,7 +489,7 @@ public class IngestManager { * content. */ void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); } /** @@ -532,100 +532,124 @@ public class IngestManager { /** * Creates and starts an ingest job, i.e., processing by ingest modules, for - * a data source. + * each data source in a collection of data sources. */ - private final class StartIngestJobsCallable implements Callable { + private final class IngestJobsStarter implements Callable { private final long threadId; - private final Content dataSource; + private final Collection dataSources; private final IngestJobSettings settings; private final boolean doStartupErrorsMsgBox; private ProgressHandle progress; - StartIngestJobsCallable(long threadId, Content dataSource, IngestJobSettings settings, boolean doStartupErrorsMsgBox) { + IngestJobsStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { this.threadId = threadId; - this.dataSource = dataSource; + this.dataSources = dataSources; this.settings = settings; - this.doStartupErrorsMsgBox = doStartupErrorsMsgBox; + this.doStartupErrorsMsgBox = doMessageDialogs; } @Override public Void call() { + if (this.dataSources.isEmpty() || Thread.currentThread().isInterrupted()) { + return null; + } + + /** + * Set up a progress bar. + */ + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.displayName"); + progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (progress != null) { + progress.setDisplayName(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.cancelling", + displayName)); + } + Future handle = ingestJobStarters.remove(threadId); + handle.cancel(true); + return true; + } + }); + progress.start(dataSources.size()); + try { - if (Thread.currentThread().isInterrupted()) { - return null; - } - - /** - * Set up a progress bar. - */ - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.displayName"); - progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - if (progress != null) { - progress.setDisplayName(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.cancelling", - displayName)); + int workUnitsCompleted = 0; + for (Content dataSource : this.dataSources) { + try { + if (Thread.currentThread().isInterrupted()) { + return null; } - Future handle = startIngestJobsCallables.remove(threadId); - handle.cancel(true); - return true; - } - }); - progress.switchToIndeterminate(); - progress.start(); - /** - * Create and start an ingest job for the data source. - */ - List errors = IngestJob.startJob(dataSource, this.settings); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { - // Report the errors to the user. They have already been logged. - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); + String progressMessage = NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.progressingDisplayName", + dataSource.getName()); + progress.progress(progressMessage); + + /** + * Start an ingest job for the data source. + */ + List errors = IngestJob.startJob(dataSource, this.settings); + if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); + } + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); + } + + progress.progress(progressMessage, ++workUnitsCompleted); + + } catch (Throwable ex) { + /** + * This is an exceptions firewall. + */ + logger.log(Level.SEVERE, "Failed to create ingest job for " + dataSource.getName(), ex); //NON-NLS + if (this.doStartupErrorsMsgBox) { + JOptionPane.showMessageDialog(null, ex.getLocalizedMessage(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); + } } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); } - } catch (Exception ex) { - logger.log(Level.SEVERE, "Failed to create ingest job", ex); //NON-NLS } finally { progress.finish(); - startIngestJobsCallables.remove(threadId); + ingestJobStarters.remove(threadId); } return null; } + } /** * A consumer for an ingest task queue. */ - private final class ExecuteIngestTasksRunnable implements Runnable { + private final class IngestTaskExecuter implements Runnable { private final long threadId; private final IngestTaskQueue tasks; - ExecuteIngestTasksRunnable(long threadId, IngestTaskQueue tasks) { + IngestTaskExecuter(long threadId, IngestTaskQueue tasks) { this.threadId = threadId; this.tasks = tasks; } @@ -649,7 +673,7 @@ public class IngestManager { /** * Fires ingest events to ingest manager property change listeners. */ - private static final class FireIngestEventRunnable implements Runnable { + private static final class IngestEventPublisher implements Runnable { private final PropertyChangeSupport publisher; private final IngestJobEvent jobEvent; @@ -657,7 +681,7 @@ public class IngestManager { private final Object oldValue; private final Object newValue; - FireIngestEventRunnable(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) { + IngestEventPublisher(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) { this.publisher = publisher; this.jobEvent = event; this.moduleEvent = null; @@ -665,7 +689,7 @@ public class IngestManager { this.newValue = newValue; } - FireIngestEventRunnable(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) { + IngestEventPublisher(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) { this.publisher = publisher; this.jobEvent = null; this.moduleEvent = event; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index f726a1d525..d8a7536ac1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -198,14 +198,9 @@ public final class RunIngestModulesDialog extends JDialog { IngestJobSettings ingestJobSettings = this.ingestJobSettingsPanel.getSettings(); ingestJobSettings.save(); showWarnings(ingestJobSettings); - if (startIngestJob) { - IngestManager ingestManager = IngestManager.getInstance(); - for (Content dataSource : RunIngestModulesDialog.this.dataSources) { - ingestManager.startIngestJob(dataSource, ingestJobSettings, true); - } + IngestManager.getInstance().startIngestJobs(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); } - setVisible(false); dispose(); } From 41bfa204a2c8413f61a1a66bf2ba5b1ee2fdd0e9 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 8 Dec 2014 23:58:10 -0500 Subject: [PATCH 42/84] Improve restoration batching of data sources to ingest jobs start API --- .../autopsy/ingest/IngestManager.java | 194 +++++++++--------- 1 file changed, 101 insertions(+), 93 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index e2f553ca3c..a336dd9bd9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -47,8 +47,8 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** - * Manages the creation and execution of ingest jobs, i.e., processing of data - * sources by ingest modules. + * Manages the creation and execution of ingest jobs, i.e., the processing of + * data sources by ingest modules. */ public class IngestManager { @@ -130,9 +130,10 @@ public class IngestManager { }; /** - * Gets the ingest manager. + * Gets the manager of the creation and execution of ingest jobs, i.e., the + * processing of data sources by ingest modules. * - * @return A singleton IngestManager object. + * @return A singleton ingest manager object. */ public synchronized static IngestManager getInstance() { if (instance == null) { @@ -157,8 +158,8 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for a - * collection of data sources. + * Starts an ingest job, i.e., processing by ingest modules, for each data + * source in a collection of data sources. * * @param dataSources The data sources to be processed. * @param settings The ingest job settings. @@ -179,7 +180,7 @@ public class IngestManager { } /** - * Queries whether any ingest jobs are in progress. + * Queries whether or not any ingest jobs are in progress. * * @return True or false. */ @@ -250,7 +251,7 @@ public class IngestManager { } /** - * Remove an ingest job and ingest module event property change listener. + * Removes an ingest job and ingest module event property change listener. * * @param listener The PropertyChangeListener to unregister. * @deprecated Use removeIngestJobEventListener() and/or @@ -263,10 +264,11 @@ public class IngestManager { } /** - * Starts the ingest monitor and submits task execution tasks (Callable - * objects) to the data source ingest and file ingest thread pools. The task - * execution tasks are simple consumers that will normally run as long as - * the application runs + * Constructs a manager of the creation and execution of ingest jobs, i.e., + * the processing of data sources by ingest modules. The manager immediately + * submits ingest task executers (Callable objects) to the data source level + * ingest and file level ingest thread pools. The ingest task executers are + * simple consumers that will normally run as long as the application runs. */ private IngestManager() { startDataSourceIngestThread(); @@ -292,7 +294,7 @@ public class IngestManager { } /** - * Submits a ExecuteIngestTasksTask Callable to the data source ingest task + * Submits an ingest task executer Callable to the data source level ingest * thread pool. */ private void startDataSourceIngestThread() { @@ -302,8 +304,8 @@ public class IngestManager { } /** - * Submits a ExecuteIngestTasksTask Callable to the data source ingest - * thread pool. + * Submits a ingest task executer Callable to the file level ingest thread + * pool. */ private void startFileIngestThread() { long threadId = nextThreadId.incrementAndGet(); @@ -551,94 +553,100 @@ public class IngestManager { @Override public Void call() { - if (this.dataSources.isEmpty() || Thread.currentThread().isInterrupted()) { - return null; - } - - /** - * Set up a progress bar. - */ - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.displayName"); - progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - if (progress != null) { - progress.setDisplayName(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.cancelling", - displayName)); - } - Future handle = ingestJobStarters.remove(threadId); - handle.cancel(true); - return true; - } - }); - progress.start(dataSources.size()); - try { + /** + * Bail out if there is nothing to do or cancellation has been + * requested. + */ + if (this.dataSources.isEmpty() || Thread.currentThread().isInterrupted()) { + return null; + } + + /** + * Set up a progress bar. + */ + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.displayName"); + progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (progress != null) { + progress.setDisplayName(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.cancelling", + displayName)); + } + Future handle = ingestJobStarters.remove(threadId); + handle.cancel(true); + return true; + } + }); + progress.start(dataSources.size()); + + /** + * Try to start the ingest jobs. + */ int workUnitsCompleted = 0; for (Content dataSource : this.dataSources) { - try { - if (Thread.currentThread().isInterrupted()) { - return null; - } - String progressMessage = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.progressingDisplayName", - dataSource.getName()); - progress.progress(progressMessage); - - /** - * Start an ingest job for the data source. - */ - List errors = IngestJob.startJob(dataSource, this.settings); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); - } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - } - - progress.progress(progressMessage, ++workUnitsCompleted); - - } catch (Throwable ex) { - /** - * This is an exceptions firewall. - */ - logger.log(Level.SEVERE, "Failed to create ingest job for " + dataSource.getName(), ex); //NON-NLS - if (this.doStartupErrorsMsgBox) { - JOptionPane.showMessageDialog(null, ex.getLocalizedMessage(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - } + /** + * Cancellation check. + */ + if (Thread.currentThread().isInterrupted()) { + return null; } + + /** + * Add a "subtitle" to the display name of the progress bar + * to indicate an ingest job is being started for this data + * source. + */ + String progressMessage = NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.progressingDisplayName", + dataSource.getName()); + progress.progress(progressMessage); + + /** + * Start an ingest job for this data source. + */ + List errors = IngestJob.startJob(dataSource, this.settings); + if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); + } + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); + } + + progress.progress(progressMessage, ++workUnitsCompleted); } + + return null; + } finally { - progress.finish(); + if (null != progress) { + progress.finish(); + } ingestJobStarters.remove(threadId); } - - return null; } - + } /** From 51f080de9aef1902615008dc4e07f26f6d1cca56 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 9 Dec 2014 09:55:31 -0500 Subject: [PATCH 43/84] Remove dead code from IngestManager and start refactoring --- .../autopsy/ingest/IngestManager.java | 38 +++++-------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index a336dd9bd9..3027002b3f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -70,11 +70,18 @@ public class IngestManager { private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); - private final Object processedFilesSnapshotLock = new Object(); - private ProcessedFilesSnapshot processedFilesSnapshot = new ProcessedFilesSnapshot(); private volatile IngestMessageTopComponent ingestMessageBox; private int numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; + /** + * These static fields are used for the creation and management of ingest + * jobs in progress. + */ + private static volatile boolean jobCreationIsEnabled; + private static final AtomicLong nextJobId = new AtomicLong(0L); + private static final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); + + /** * Ingest job events. */ @@ -388,10 +395,6 @@ public class IngestManager { IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId()); ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); - synchronized (processedFilesSnapshotLock) { - processedFilesSnapshot.incrementProcessedFilesCount(); - } - incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } @@ -782,27 +785,4 @@ public class IngestManager { } } - static final class ProcessedFilesSnapshot { - - private final Date startTime; - private long processedFilesCount; - - ProcessedFilesSnapshot() { - this.startTime = new Date(); - this.processedFilesCount = 0; - } - - void incrementProcessedFilesCount() { - ++processedFilesCount; - } - - Date getStartTime() { - return startTime; - } - - long getProcessedFilesCount() { - return processedFilesCount; - } - } - } From fb12ca7adc309e087538ff09eed750771c2d44b0 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 9 Dec 2014 14:38:22 -0500 Subject: [PATCH 44/84] Refactor IngestJob and IngestManager in preparation for API work --- .../sleuthkit/autopsy/ingest/IngestJob.java | 135 ++++------------ .../autopsy/ingest/IngestManager.java | 147 ++++++++++++++---- .../ingest/IngestProgressSnapshotPanel.java | 2 +- 3 files changed, 154 insertions(+), 130 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index cd1e6e3480..98dbbb3930 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; @@ -45,35 +44,16 @@ final class IngestJob { private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); /** - * The task scheduler singleton is responsible for creating and scheduling - * the ingest tasks that make up ingest jobs. - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - - /** - * These static fields are used for the creation and management of ingest - * jobs in progress. - */ - private static volatile boolean jobCreationIsEnabled; - private static final AtomicLong nextJobId = new AtomicLong(0L); - private static final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); - - /** - * These fields define the ingest job, including its ingest pipelines. Note - * that there is a collection of multiple copies of the file ingest - * pipeline, one for each file ingest thread. + * These fields define the ingest job: a unique ID supplied by the ingest + * manager, the user's ingest job settings, and the data source to be + * processed. */ private final long id; - private final Content dataSource; private final IngestJobSettings ingestJobSettings; - private final Object dataSourceIngestPipelineLock; - private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; - private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; - private final LinkedBlockingQueue fileIngestPipelines; + private final Content dataSource; /** - * An ingest runs in stages. + * An ingest job runs in stages. */ private static enum Stages { @@ -99,6 +79,28 @@ final class IngestJob { private Stages stage; private final Object stageCompletionCheckLock; + /** + * There is a data source level ingest modules pipeline for both the first + * stage and the second stage. Longer running, lower priority data source + * level ingest modules belong in the second stage pipeline. + */ + private final Object dataSourceIngestPipelineLock; + private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; + private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; + private DataSourceIngestPipeline currentDataSourceIngestPipeline; + + /** + * There is a collection of identical file level ingest module pipelines, + * one for each file level ingest thread in the ingest manager. + */ + private final LinkedBlockingQueue fileIngestPipelines; + + /** + * The task scheduler singleton is responsible for creating and scheduling + * the ingest tasks that make up this ingest jobs. + */ + private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + /** * These fields are used to provide data source level task progress bars for * the job. @@ -128,70 +130,6 @@ final class IngestJob { */ private final long startTime; - /** - * Enables and disables ingest job creation. - * - * @param enabled True or false. - */ - static void jobCreationEnabled(boolean enabled) { - IngestJob.jobCreationIsEnabled = enabled; - } - - /** - * Starts an ingest job for a data source. - * - * @param dataSource The data source to ingest. - * @param settings The settings for the job. - * @return A collection of ingest module start up errors, empty on success. - */ - static List startJob(Content dataSource, IngestJobSettings settings) { - List errors = new ArrayList<>(); - if (IngestJob.jobCreationIsEnabled) { - long jobId = nextJobId.incrementAndGet(); - IngestJob job = new IngestJob(jobId, dataSource, settings); - IngestJob.jobsById.put(jobId, job); - errors = job.start(settings.getEnabledIngestModuleTemplates()); - if (errors.isEmpty() && job.hasIngestPipeline()) { - IngestManager.getInstance().fireIngestJobStarted(jobId); - IngestJob.logger.log(Level.INFO, "Ingest job {0} started", jobId); - } else { - IngestJob.jobsById.remove(jobId); - } - } - return errors; - } - - /** - * Queries whether or not ingest jobs are running. - * - * @return True or false. - */ - static boolean ingestJobsAreRunning() { - return !jobsById.isEmpty(); - } - - /** - * Gets snapshots of the state of all running ingest jobs. - * - * @return A list of ingest job state snapshots. - */ - static List getJobSnapshots() { - List snapShots = new ArrayList<>(); - for (IngestJob job : IngestJob.jobsById.values()) { - snapShots.add(job.getSnapshot()); - } - return snapShots; - } - - /** - * Cancels all running ingest jobs. - */ - static void cancelAllJobs() { - for (IngestJob job : jobsById.values()) { - job.cancel(); - } - } - /** * Constructs an ingest job. * @@ -200,10 +138,9 @@ final class IngestJob { * @param processUnallocatedSpace Whether or not unallocated space should be * processed during the ingest job. */ - private IngestJob(long id, Content dataSource, IngestJobSettings settings) { + IngestJob(long id, Content dataSource, IngestJobSettings settings) { this.id = id; this.dataSource = dataSource; - this.ingestJobSettings = settings; this.dataSourceIngestPipelineLock = new Object(); this.fileIngestPipelines = new LinkedBlockingQueue<>(); @@ -575,7 +512,7 @@ final class IngestJob { * * @return A collection of ingest module startup errors, empty on success. */ - private List start(List ingestModuleTemplates) { + List start(List ingestModuleTemplates) { this.createIngestPipelines(ingestModuleTemplates); List errors = startUpIngestPipelines(); if (errors.isEmpty()) { @@ -738,7 +675,7 @@ final class IngestJob { * * @return True or false. */ - private boolean hasIngestPipeline() { + boolean hasIngestPipeline() { return this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline() || this.hasSecondStageDataSourceIngestPipeline(); @@ -952,14 +889,7 @@ final class IngestJob { } } - IngestJob.jobsById.remove(this.id); - if (!this.isCancelled()) { - logger.log(Level.INFO, "Ingest job {0} completed", this.id); - IngestManager.getInstance().fireIngestJobCompleted(this.id); - } else { - logger.log(Level.INFO, "Ingest job {0} cancelled", this.id); - IngestManager.getInstance().fireIngestJobCancelled(this.id); - } + IngestManager.getInstance().finishJob(this); } /** @@ -986,9 +916,8 @@ final class IngestJob { * * @return An ingest job statistics object. */ - private IngestJobSnapshot getSnapshot() { + IngestJobSnapshot getSnapshot() { return new IngestJobSnapshot(); - } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3027002b3f..20480b92a9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -52,36 +52,80 @@ import org.sleuthkit.datamodel.Content; */ public class IngestManager { + private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); + private static IngestManager instance = null; + + /** + * When ingest job creation is enabled, the ingest manager assigns a unique + * ID to each ingest job and maintains a mapping of job IDs to jobs. + */ + private volatile boolean jobCreationIsEnabled; + private final AtomicLong nextJobId = new AtomicLong(0L); + private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); + + /** + * Each runnable/callable task the ingest manager farms out to a thread pool + * is given a unique thread/task ID. + */ + private final AtomicLong nextThreadId = new AtomicLong(0L); + + /** + * Ingest jobs are started on a pool thread by ingest job starters. A + * mapping of thread/task IDs to the result objects associated with each + * ingest job starter is maintained to provide handles that can be used to + * cancel the ingest job starter. + */ + private final ConcurrentHashMap> ingestJobStarters = new ConcurrentHashMap<>(); + private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); + + /** + * Ingest jobs use an ingest task scheduler to break themselves down into + * data source level and file level tasks. The ingest scheduler puts these + * ingest tasks into queues for execution on ingest manager pool threads by + * ingest task executers. There is a single data source level ingest thread + * and a user configurable number of file level ingest threads. + */ private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; - private static final int MAX_ERROR_MESSAGE_POSTS = 200; - private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); - private static IngestManager instance = null; - private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); - private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); - private final IngestMonitor ingestMonitor = new IngestMonitor(); - private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); + private int numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); private final ExecutorService fileIngestThreadPool; - private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); - private final AtomicLong nextThreadId = new AtomicLong(0L); - private final ConcurrentHashMap> ingestJobStarters = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. - private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); - private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. - private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); - private volatile IngestMessageTopComponent ingestMessageBox; - private int numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; /** - * These static fields are used for the creation and management of ingest - * jobs in progress. + * The ingest manager uses the property change feature from Java Beans as an + * event publishing mechanism. There are two kinds of events, ingest job + * events and ingest module events. Property changes are fired by ingest + * event publishers on a pool thread. */ - private static volatile boolean jobCreationIsEnabled; - private static final AtomicLong nextJobId = new AtomicLong(0L); - private static final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); + private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); + private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); + private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); + + /** + * The ingest manager uses an ingest monitor to determine when system + * resources are under pressure. If the monitor detects such a situation, it + * calls back to the ingest manager to cancel all ingest jobs in progress. + */ + private final IngestMonitor ingestMonitor = new IngestMonitor(); + + /** + * The ingest manager provides access to a top component that is used by + * ingest module to post messages for the user. A count of the posts is used + * as a cap to avoid bogging down the application. + */ + private static final int MAX_ERROR_MESSAGE_POSTS = 200; + private volatile IngestMessageTopComponent ingestMessageBox; + private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); + + /** + * The ingest manager supports reporting of ingest processing progress by + * collecting snapshots of the activities of the ingest threads, ingest job + * progress, and ingest module run times. + */ + private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. + private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); - /** * Ingest job events. */ @@ -192,7 +236,7 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { - return IngestJob.ingestJobsAreRunning(); + return !this.jobsById.isEmpty(); } /** @@ -205,7 +249,9 @@ public class IngestManager { } // Cancel all the jobs already created. - IngestJob.cancelAllJobs(); + for (IngestJob job : this.jobsById.values()) { + job.cancel(); + } } /** @@ -336,12 +382,12 @@ public class IngestManager { } void handleCaseOpened() { - IngestJob.jobCreationEnabled(true); + this.jobCreationIsEnabled = true; clearIngestMessageBox(); } void handleCaseClosed() { - IngestJob.jobCreationEnabled(false); + this.jobCreationIsEnabled = false; cancelAllIngestJobs(); clearIngestMessageBox(); } @@ -353,6 +399,42 @@ public class IngestManager { ingestErrorMessagePosts.set(0); } + /** + * Starts an ingest job for a data source. + * + * @param dataSource The data source to ingest. + * @param settings The settings for the job. + * @return A collection of ingest module start up errors, empty on success. + */ + private List startJob(Content dataSource, IngestJobSettings settings) { + List errors = new ArrayList<>(); + if (this.jobCreationIsEnabled) { + long jobId = this.nextJobId.incrementAndGet(); + IngestJob job = new IngestJob(jobId, dataSource, settings); + this.jobsById.put(jobId, job); + errors = job.start(settings.getEnabledIngestModuleTemplates()); + if (errors.isEmpty() && job.hasIngestPipeline()) { + this.fireIngestJobStarted(jobId); + IngestManager.logger.log(Level.INFO, "Ingest job {0} started", jobId); + } else { + this.jobsById.remove(jobId); + } + } + return errors; + } + + void finishJob(IngestJob job) { + long jobId = job.getId(); + this.jobsById.remove(jobId); + if (!job.isCancelled()) { + IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); + this.fireIngestJobCompleted(jobId); + } else { + IngestManager.logger.log(Level.INFO, "Ingest job {0} cancelled", jobId); + this.fireIngestJobCancelled(jobId); + } + } + /** * Called each time a module in a data source pipeline starts * @@ -535,6 +617,19 @@ public class IngestManager { } } + /** + * Gets snapshots of the state of all running ingest jobs. + * + * @return A list of ingest job state snapshots. + */ + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); + for (IngestJob job : this.jobsById.values()) { + snapShots.add(job.getSnapshot()); + } + return snapShots; + } + /** * Creates and starts an ingest job, i.e., processing by ingest modules, for * each data source in a collection of data sources. @@ -611,7 +706,7 @@ public class IngestManager { /** * Start an ingest job for this data source. */ - List errors = IngestJob.startJob(dataSource, this.settings); + List errors = IngestManager.this.startJob(dataSource, this.settings); if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { StringBuilder moduleStartUpErrors = new StringBuilder(); for (IngestModuleError error : errors) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 792a6a4e7f..47042c8335 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -170,7 +170,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { } private void refresh() { - jobSnapshots = IngestJob.getJobSnapshots(); + jobSnapshots = IngestManager.getInstance().getIngestJobSnapshots(); fireTableDataChanged(); } From 194963862e8a7fd24b170af47c49e229964797cf Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 10 Dec 2014 09:26:20 -0500 Subject: [PATCH 45/84] Add comment to IngestManager --- .../sleuthkit/autopsy/ingest/IngestManager.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 20480b92a9..3e3f70c473 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -56,10 +56,9 @@ public class IngestManager { private static IngestManager instance = null; /** - * When ingest job creation is enabled, the ingest manager assigns a unique - * ID to each ingest job and maintains a mapping of job IDs to jobs. + * The ingest manager assigns a unique ID to each ingest job and maintains a + * mapping of job IDs to jobs. */ - private volatile boolean jobCreationIsEnabled; private final AtomicLong nextJobId = new AtomicLong(0L); private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); @@ -126,6 +125,12 @@ public class IngestManager { private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); + /** + * The ingest job creation capability of the ingest manager can be turned on + * and off to support an orderly shut down of the application. + */ + private volatile boolean jobCreationIsEnabled; + /** * Ingest job events. */ @@ -425,7 +430,7 @@ public class IngestManager { void finishJob(IngestJob job) { long jobId = job.getId(); - this.jobsById.remove(jobId); + this.jobsById.remove(jobId); if (!job.isCancelled()) { IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); this.fireIngestJobCompleted(jobId); @@ -434,7 +439,7 @@ public class IngestManager { this.fireIngestJobCancelled(jobId); } } - + /** * Called each time a module in a data source pipeline starts * @@ -629,7 +634,7 @@ public class IngestManager { } return snapShots; } - + /** * Creates and starts an ingest job, i.e., processing by ingest modules, for * each data source in a collection of data sources. From f71063eb569700628e789a2c9414631c3dd28cf7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 10 Dec 2014 15:34:40 -0500 Subject: [PATCH 46/84] Improve INgestManager and IngestJob documentation --- .../sleuthkit/autopsy/ingest/IngestJob.java | 58 +++++++++---------- .../autopsy/ingest/IngestManager.java | 9 +-- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 98dbbb3930..ff463321a7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -23,7 +23,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.Level; import javax.swing.JOptionPane; @@ -44,7 +43,7 @@ final class IngestJob { private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); /** - * These fields define the ingest job: a unique ID supplied by the ingest + * These fields define an ingest job: a unique ID supplied by the ingest * manager, the user's ingest job settings, and the data source to be * processed. */ @@ -80,9 +79,9 @@ final class IngestJob { private final Object stageCompletionCheckLock; /** - * There is a data source level ingest modules pipeline for both the first - * stage and the second stage. Longer running, lower priority data source - * level ingest modules belong in the second stage pipeline. + * An ingest job has separate data source level ingest module pipelines for + * each processing stage. Longer running, lower priority modules belong in + * the second stage pipeline. */ private final Object dataSourceIngestPipelineLock; private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; @@ -90,27 +89,34 @@ final class IngestJob { private DataSourceIngestPipeline currentDataSourceIngestPipeline; /** - * There is a collection of identical file level ingest module pipelines, - * one for each file level ingest thread in the ingest manager. + * An ingest job has a collection of identical file level ingest module + * pipelines, one for each file level ingest thread in the ingest manager. */ - private final LinkedBlockingQueue fileIngestPipelines; - + private final LinkedBlockingQueue fileIngestPipelines; + /** - * The task scheduler singleton is responsible for creating and scheduling - * the ingest tasks that make up this ingest jobs. + * An ingest job supports cancellation of either the currently running data + * source level ingest module or the entire ingest job. + */ + private volatile boolean currentDataSourceIngestModuleCancelled; + private volatile boolean cancelled; + + /** + * An ingest job uses the task scheduler singleton to create and queue the + * ingest tasks that make up the job. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - + /** - * These fields are used to provide data source level task progress bars for - * the job. + * These fields are used to report data source level ingest progress for the + * ingest job. */ private final Object dataSourceIngestProgressLock; private ProgressHandle dataSourceIngestProgress; /** - * These fields are used to provide file level ingest task progress bars for - * the job. + * These fields are used to report file level ingest task progress for the + * ingest job. */ private final Object fileIngestProgressLock; private final List filesInProgress; @@ -119,24 +125,16 @@ final class IngestJob { private ProgressHandle fileIngestProgress; /** - * These fields support cancellation of either the currently running data - * source level ingest module or the entire ingest job. + * This field is used to record the creation of the ingest job. */ - private volatile boolean currentDataSourceIngestModuleCancelled; - private volatile boolean cancelled; - - /** - * This field is used for generating ingest job diagnostic data. - */ - private final long startTime; + private final long createTime; /** * Constructs an ingest job. * * @param id The identifier assigned to the job. * @param dataSource The data source to be ingested. - * @param processUnallocatedSpace Whether or not unallocated space should be - * processed during the ingest job. + * @param settings The settings for the ingest job. */ IngestJob(long id, Content dataSource, IngestJobSettings settings) { this.id = id; @@ -149,7 +147,7 @@ final class IngestJob { this.fileIngestProgressLock = new Object(); this.stage = IngestJob.Stages.INITIALIZATION; this.stageCompletionCheckLock = new Object(); - this.startTime = new Date().getTime(); + this.createTime = new Date().getTime(); } /** @@ -934,13 +932,13 @@ final class IngestJob { private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; /** - * Constructs an object to stores basic diagnostic statistics for an + * Constructs an object to store basic diagnostic statistics for an * ingest job. */ IngestJobSnapshot() { this.jobId = IngestJob.this.id; this.dataSource = IngestJob.this.dataSource.getName(); - this.startTime = IngestJob.this.startTime; + this.startTime = IngestJob.this.createTime; synchronized (IngestJob.this.fileIngestProgressLock) { this.processedFiles = IngestJob.this.processedFiles; this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3e3f70c473..7d32adbcd3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -63,11 +63,12 @@ public class IngestManager { private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); /** - * Each runnable/callable task the ingest manager farms out to a thread pool - * is given a unique thread/task ID. + * Each runnable/callable task the ingest manager submits to its thread + * pools is given a unique thread/task ID. */ - private final AtomicLong nextThreadId = new AtomicLong(0L); - + // TODO: It is no longer necessary to have multiple thread pools. + private final AtomicLong nextThreadId = new AtomicLong(0L); + /** * Ingest jobs are started on a pool thread by ingest job starters. A * mapping of thread/task IDs to the result objects associated with each From be4490650f104e1dc832d60dcb1a8e0549450d10 Mon Sep 17 00:00:00 2001 From: Karl Mortensen Date: Fri, 12 Dec 2014 09:11:00 -0500 Subject: [PATCH 47/84] Update IngestProgressSnapshotPanel and StartupWindow --- .../ShowIngestProgressSnapshotAction.java | 1 - .../autopsy/casemodule/StartupWindow.java | 3 +- .../ingest/IngestProgressSnapshotDialog.java | 39 ++++++++++++++++++- .../ingest/IngestProgressSnapshotPanel.java | 2 +- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java b/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java index fcedf212b2..9a9e1995e3 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ShowIngestProgressSnapshotAction.java @@ -46,7 +46,6 @@ public final class ShowIngestProgressSnapshotAction extends SystemAction impleme @Override public void actionPerformed(ActionEvent e) { IngestProgressSnapshotDialog dialog = new IngestProgressSnapshotDialog(); - dialog.setVisible(true); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/StartupWindow.java b/Core/src/org/sleuthkit/autopsy/casemodule/StartupWindow.java index 4c99d4f582..b68c55d013 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/StartupWindow.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/StartupWindow.java @@ -29,6 +29,7 @@ import javax.swing.JFrame; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; +import org.openide.windows.WindowManager; /** * The default implementation of the Autopsy startup window @@ -41,7 +42,7 @@ public final class StartupWindow extends JDialog implements StartupWindowInterfa private static Dimension DIMENSIONS = new Dimension(750, 400); public StartupWindow() { - super(new JFrame(TITLE), TITLE, true); + super(WindowManager.getDefault().getMainWindow(), TITLE, true); init(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java index 3d4af0b410..0e59b3ceb5 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java @@ -19,10 +19,14 @@ package org.sleuthkit.autopsy.ingest; import java.awt.BorderLayout; +import java.awt.Container; +import java.awt.Dialog; import java.awt.Dimension; import java.awt.Toolkit; +import java.awt.Window; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; import javax.swing.JDialog; -import javax.swing.JFrame; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; @@ -33,12 +37,40 @@ public final class IngestProgressSnapshotDialog extends JDialog { private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestProgressSnapshotDialog.title.text"); private static final Dimension DIMENSIONS = new Dimension(500, 300); + private JDialog pseudoOwner = null; /** * Constructs a non-modal instance of the dialog with its own frame. */ public IngestProgressSnapshotDialog() { - super((JFrame) WindowManager.getDefault().getMainWindow(), TITLE, false); + this((Window) WindowManager.getDefault().getMainWindow(), false); + } + + /** + * Constructs an instance of the dialog with its own frame. Could be modal. + * + * @param owner - the owner of this dialog. If this dialog should be modal, the owner gets set to non modal. + * @param shouldBeModal - true if this should be modal, false otherwise. + */ + public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) { + super((Window) owner, TITLE, ModalityType.MODELESS); + if (shouldBeModal) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. + pseudoOwner = (JDialog) owner; + addWindowListener(new WindowAdapter() { + @Override + public void windowClosed(WindowEvent e) { // Put it back to how it was before we manipulated it. + pseudoOwner.setVisible(false); + pseudoOwner.setModalityType(Dialog.ModalityType.APPLICATION_MODAL); + pseudoOwner.toFront(); + pseudoOwner.setVisible(true); + } + }); + pseudoOwner.setVisible(false); + pseudoOwner.setModalityType(Dialog.ModalityType.MODELESS); + pseudoOwner.toFront(); + pseudoOwner.repaint(); + pseudoOwner.setVisible(true); + } setResizable(true); setLayout(new BorderLayout()); Dimension screenDimension = Toolkit.getDefaultToolkit().getScreenSize(); @@ -49,6 +81,9 @@ public final class IngestProgressSnapshotDialog extends JDialog { add(new IngestProgressSnapshotPanel(this)); pack(); setResizable(false); + if (shouldBeModal) { // if called from a modal dialog, become modal, otherwise don't. + setModal(true); + } setVisible(true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 47042c8335..0d05337499 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -429,7 +429,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { }// //GEN-END:initComponents private void closeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_closeButtonActionPerformed - parent.setVisible(false); + parent.dispose(); }//GEN-LAST:event_closeButtonActionPerformed private void refreshButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_refreshButtonActionPerformed From b2cac4427459e0a542be077c31f82db540c15b02 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 12 Dec 2014 17:04:20 -0500 Subject: [PATCH 48/84] Refactoring to produce new IngestJob class, old class => DataSourceIngestJon --- .../autopsy/ingest/DataSourceIngestJob.java | 1063 ++++++++++++++++ .../DataSourceIngestModuleProgress.java | 4 +- .../ingest/DataSourceIngestPipeline.java | 116 +- .../autopsy/ingest/DataSourceIngestTask.java | 2 +- .../autopsy/ingest/FileIngestPipeline.java | 4 +- .../autopsy/ingest/FileIngestTask.java | 6 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 1081 ++--------------- .../autopsy/ingest/IngestJobContext.java | 4 +- .../autopsy/ingest/IngestManager.java | 115 +- .../ingest/IngestProgressSnapshotPanel.java | 4 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 6 +- .../autopsy/ingest/IngestTasksScheduler.java | 12 +- 12 files changed, 1355 insertions(+), 1062 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java new file mode 100644 index 0000000000..254510bb6c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -0,0 +1,1063 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import javax.swing.JOptionPane; +import org.netbeans.api.progress.ProgressHandle; +import org.netbeans.api.progress.ProgressHandleFactory; +import org.openide.util.Cancellable; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; + +/** + * Encapsulates a data source to be processed and the settings, ingest module + * pipelines, and progress bars that are used to process it. + */ +final class DataSourceIngestJob { + + private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName()); + private static final AtomicLong nextJobId = new AtomicLong(0L); + + /** + * These fields define a data source ingest job: the parent ingest job, a + * unique ID, the user's ingest job settings, and the data source to be + * processed. + */ + private final IngestJob parentJob; + private final long id; + private final IngestJobSettings settings; + private final Content dataSource; + + /** + * An ingest job runs in stages. + */ + private static enum Stages { + + /** + * Setting up for processing. + */ + INITIALIZATION, + /** + * Running high priority data source level ingest modules and file level + * ingest modules. + */ + FIRST, + /** + * Running lower priority, usually long-running, data source level + * ingest modules. + */ + SECOND, + /** + * Cleaning up. + */ + FINALIZATION + }; + private Stages stage; + private final Object stageCompletionCheckLock; + + /** + * An ingest job has separate data source level ingest module pipelines for + * each processing stage. Longer running, lower priority modules belong in + * the second stage pipeline. + */ + private final Object dataSourceIngestPipelineLock; + private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; + private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; + private DataSourceIngestPipeline currentDataSourceIngestPipeline; + + /** + * An ingest job has a collection of identical file level ingest module + * pipelines, one for each file level ingest thread in the ingest manager. + */ + private final LinkedBlockingQueue fileIngestPipelines; + + /** + * An ingest job supports cancellation of either the currently running data + * source level ingest module or the entire ingest job. + */ + private volatile boolean currentDataSourceIngestModuleCancelled; + private volatile boolean cancelled; + + /** + * An ingest job uses the task scheduler singleton to create and queue the + * ingest tasks that make up the job. + */ + private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + + /** + * These fields are used to report data source level ingest progress for the + * ingest job. + */ + private final Object dataSourceIngestProgressLock; + private ProgressHandle dataSourceIngestProgress; + + /** + * These fields are used to report file level ingest task progress for the + * ingest job. + */ + private final Object fileIngestProgressLock; + private final List filesInProgress; + private long estimatedFilesToProcess; + private long processedFiles; + private ProgressHandle fileIngestProgress; + + /** + * This field is used to record the creation of the ingest job. + */ + private final long createTime; + + /** + * Constructs an ingest job. + * + * @param parentJob The ingest job of which this data source ingest job is a + * part. + * @param dataSource The data source to be ingested. + * @param settings The settings for the ingest job. + */ + DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings) { + this.parentJob = parentJob; + this.id = DataSourceIngestJob.nextJobId.getAndIncrement(); + this.dataSource = dataSource; + this.settings = settings; + this.dataSourceIngestPipelineLock = new Object(); + this.fileIngestPipelines = new LinkedBlockingQueue<>(); + this.filesInProgress = new ArrayList<>(); + this.dataSourceIngestProgressLock = new Object(); + this.fileIngestProgressLock = new Object(); + this.stage = DataSourceIngestJob.Stages.INITIALIZATION; + this.stageCompletionCheckLock = new Object(); + this.createTime = new Date().getTime(); + } + + /** + * Gets the identifier assigned to this job. + * + * @return The job identifier. + */ + long getId() { + return this.id; + } + + /** + * Gets the data source to be ingested by this job. + * + * @return A Content object representing the data source. + */ + Content getDataSource() { + return this.dataSource; + } + + /** + * Gets whether or not unallocated space should be processed as part of this + * job. + * + * @return True or false. + */ + boolean shouldProcessUnallocatedSpace() { + return this.settings.getProcessUnallocatedSpace(); + } + + /** + * Passes the data source for this job through the currently active data + * source level ingest pipeline. + * + * @param task A data source ingest task wrapping the data source. + */ + void process(DataSourceIngestTask task) { + try { + synchronized (this.dataSourceIngestPipelineLock) { + if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + /** + * Run the data source through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } + + /** + * Shut down the data source ingest progress bar right away. Data + * source-level processing is finished for this stage. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Passes a file from the data source for this job through the file level + * ingest pipeline. + * + * @param task A file ingest task. + * @throws InterruptedException if the thread executing this code is + * interrupted while blocked on taking from or putting to the file ingest + * pipelines collection. + */ + void process(FileIngestTask task) throws InterruptedException { + try { + if (!this.isCancelled()) { + /** + * Get a file ingest pipeline not currently in use by another + * file ingest thread. + */ + FileIngestPipeline pipeline = this.fileIngestPipelines.take(); + if (!pipeline.isEmpty()) { + /** + * Get the file to process. + */ + AbstractFile file = task.getFile(); + + /** + * Update the file ingest progress bar. + */ + synchronized (this.fileIngestProgressLock) { + ++this.processedFiles; + if (this.processedFiles <= this.estimatedFilesToProcess) { + this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); + } else { + this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); + } + this.filesInProgress.add(file.getName()); + } + + /** + * Run the file through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(pipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + /** + * Update the file ingest progress bar again, in case the + * file was being displayed. + */ + if (!this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.filesInProgress.remove(file.getName()); + if (this.filesInProgress.size() > 0) { + this.fileIngestProgress.progress(this.filesInProgress.get(0)); + } else { + this.fileIngestProgress.progress(""); + } + } + } + } + + /** + * Relinquish the pipeline so it can be reused by another file + * ingest thread. + */ + this.fileIngestPipelines.put(pipeline); + } + } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Adds more files to an ingest job, i.e., extracted or carved files. Not + * currently supported for the second stage of the job. + * + * @param files A list of files to add. + */ + void addFiles(List files) { + /** + * Note: This implementation assumes that this is being called by an an + * ingest module running code on an ingest thread that is holding a + * reference to an ingest task, so no task completion check is done. + */ + if (DataSourceIngestJob.Stages.FIRST == this.stage) { + for (AbstractFile file : files) { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); + } + } else { + DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + } + + /** + * The intended clients of this method are ingest modules running code + * on an ingest thread that is holding a reference to an ingest task, in + * which case a task completion check would not be necessary. This is a + * bit of defensive programming. + */ + this.checkForStageCompleted(); + } + + /** + * Updates the display name of the data source level ingest progress bar. + * + * @param displayName The new display name. + */ + void updateDataSourceIngestProgressBarDisplayName(String displayName) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.setDisplayName(displayName); + } + } + } + + /** + * Switches the data source level ingest progress bar to determinate mode. + * This should be called if the total work units to process the data source + * is known. + * + * @param workUnits Total number of work units for the processing of the + * data source. + */ + void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToDeterminate(workUnits); + } + } + } + } + + /** + * Switches the data source level ingest progress bar to indeterminate mode. + * This should be called if the total work units to process the data source + * is unknown. + */ + void switchDataSourceIngestProgressBarToIndeterminate() { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + } + } + + /** + * Updates the data source level ingest progress bar with the number of work + * units performed, if in the determinate mode. + * + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(int workUnits) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress("", workUnits); + } + } + } + } + + /** + * Updates the data source level ingest progress with a new task name, where + * the task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + */ + void advanceDataSourceIngestProgressBar(String currentTask) { + if (!this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress(currentTask); + } + } + } + } + + /** + * Updates the data source level ingest progress bar with a new task name + * and the number of work units performed, if in the determinate mode. The + * task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { + if (!this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.dataSourceIngestProgress.progress(currentTask, workUnits); + } + } + } + + /** + * Queries whether or not a temporary cancellation of data source level + * ingest in order to stop the currently executing data source level ingest + * module is in effect. + * + * @return True or false. + */ + boolean currentDataSourceIngestModuleIsCancelled() { + return this.currentDataSourceIngestModuleCancelled; + } + + /** + * Rescind a temporary cancellation of data source level ingest that was + * used to stop a single data source level ingest module. + */ + void currentDataSourceIngestModuleCancellationCompleted() { + this.currentDataSourceIngestModuleCancelled = false; + + /** + * A new progress bar must be created because the cancel button of the + * previously constructed component is disabled by NetBeans when the + * user selects the "OK" button of the cancellation confirmation dialog + * popped up by NetBeans when the progress bar cancel button was + * pressed. + */ + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + this.startDataSourceIngestProgressBar(); + } + } + + /** + * Requests cancellation of ingest, i.e., a shutdown of the data source + * level and file level ingest pipelines. + */ + void cancel() { + /** + * Put a cancellation message on data source level ingest progress bar, + * if it is still running. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (dataSourceIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + dataSource.getName()); + dataSourceIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), + "IngestJob.progress.cancelling", + displayName)); + } + } + + /** + * Put a cancellation message on the file level ingest progress bar, if + * it is still running. + */ + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", + displayName)); + } + } + + this.cancelled = true; + + /** + * Tell the task scheduler to cancel all pending tasks, i.e., tasks not + * not being performed by an ingest thread. + */ + DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); + this.checkForStageCompleted(); + } + + /** + * Queries whether or not cancellation of ingest i.e., a shutdown of the + * data source level and file level ingest pipelines, has been requested. + * + * @return True or false. + */ + boolean isCancelled() { + return this.cancelled; + } + + /** + * Starts up the ingest pipelines and ingest progress bars. + * + * @return A collection of ingest module startup errors, empty on success. + */ + List start() { + this.createIngestPipelines(settings.getEnabledIngestModuleTemplates()); + List errors = startUpIngestPipelines(); + if (errors.isEmpty()) { + if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { + this.startFirstStage(); + } else if (this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } + } + return errors; + } + + /** + * Creates the file and data source ingest pipelines. + * + * @param ingestModuleTemplates Ingest module templates to use to populate + * the pipelines. + */ + private void createIngestPipelines(List ingestModuleTemplates) { + /** + * Make mappings of ingest module factory class names to templates. + */ + Map dataSourceModuleTemplates = new HashMap<>(); + Map fileModuleTemplates = new HashMap<>(); + for (IngestModuleTemplate template : ingestModuleTemplates) { + if (template.isDataSourceIngestModuleTemplate()) { + dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + if (template.isFileIngestModuleTemplate()) { + fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + } + } + + /** + * Use the mappings and the ingest pipelines configuration to create + * ordered lists of ingest module templates for each ingest pipeline. + */ + IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); + List firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + + /** + * Add any module templates that were not specified in the pipelines + * configuration to an appropriate pipeline - either the first stage + * data source ingest pipeline or the file ingest pipeline. + */ + for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { + firstStageDataSourceModuleTemplates.add(template); + } + for (IngestModuleTemplate template : fileModuleTemplates.values()) { + fileIngestModuleTemplates.add(template); + } + + /** + * Construct the data source ingest pipelines. + */ + this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); + this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); + + /** + * Construct the file ingest pipelines, one per file ingest thread. + */ + try { + int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); + } + } catch (InterruptedException ex) { + /** + * The current thread was interrupted while blocked on a full queue. + * Blocking should actually never happen here, but reset the + * interrupted flag rather than just swallowing the exception. + */ + Thread.currentThread().interrupt(); + } + } + + /** + * Use an ordered list of ingest module factory class names to create an + * ordered output list of ingest module templates for an ingest pipeline. + * The ingest module templates are removed from the input collection as they + * are added to the output collection. + * + * @param ingestModuleTemplates A mapping of ingest module factory class + * names to ingest module templates. + * @param pipelineConfig An ordered list of ingest module factory class + * names representing an ingest pipeline. + * @return + */ + private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { + List templates = new ArrayList<>(); + for (String moduleClassName : pipelineConfig) { + if (ingestModuleTemplates.containsKey(moduleClassName)) { + templates.add(ingestModuleTemplates.remove(moduleClassName)); + } + } + return templates; + } + + /** + * Starts the first stage of the job. + */ + private void startFirstStage() { + this.stage = DataSourceIngestJob.Stages.FIRST; + + /** + * Start one or both of the first stage ingest progress bars. + */ + if (this.hasFirstStageDataSourceIngestPipeline()) { + this.startDataSourceIngestProgressBar(); + } + if (this.hasFileIngestPipeline()) { + this.startFileIngestProgressBar(); + } + + /** + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + } + + /** + * Schedule the first stage tasks. + */ + if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); + } else if (this.hasFirstStageDataSourceIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this); + + /** + * No data source ingest task has been scheduled for this stage, and + * it is possible, if unlikely, that no file ingest tasks were + * actually scheduled since there are files that get filtered out by + * the tasks scheduler. In this special case, an ingest thread will + * never to check for completion of this stage of the job. + */ + this.checkForStageCompleted(); + } + } + + /** + * Starts the second stage of the ingest job. + */ + private void startSecondStage() { + this.stage = DataSourceIngestJob.Stages.SECOND; + this.startDataSourceIngestProgressBar(); + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + } + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } + + /** + * Checks to see if this job has at least one ingest pipeline. + * + * @return True or false. + */ + boolean hasIngestPipeline() { + return this.hasFirstStageDataSourceIngestPipeline() + || this.hasFileIngestPipeline() + || this.hasSecondStageDataSourceIngestPipeline(); + } + + /** + * Checks to see if this job has a first stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasFirstStageDataSourceIngestPipeline() { + return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if this job has a second stage data source level ingest + * pipeline. + * + * @return True or false. + */ + private boolean hasSecondStageDataSourceIngestPipeline() { + return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if the job has a file level ingest pipeline. + * + * @return True or false. + */ + private boolean hasFileIngestPipeline() { + return (this.fileIngestPipelines.peek().isEmpty() == false); + } + + /** + * Starts up each of the file and data source level ingest modules to + * collect possible errors. + * + * @return A collection of ingest module startup errors, empty on success. + */ + private List startUpIngestPipelines() { + List errors = new ArrayList<>(); + + // Start up the first stage data source ingest pipeline. + errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); + + // Start up the second stage data source ingest pipeline. + errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); + + // Start up the file ingest pipelines (one per file ingest thread). + for (FileIngestPipeline pipeline : this.fileIngestPipelines) { + errors.addAll(pipeline.startUp()); + if (!errors.isEmpty()) { + // If there are start up errors, the ingest job will not proceed + // and the errors will ultimately be reported to the user for + // possible remedy so shut down the pipelines now that an + // attempt has been made to start up the data source ingest + // pipeline and at least one copy of the file ingest pipeline. + // pipeline. There is no need to complete starting up all of the + // file ingest pipeline copies since any additional start up + // errors are likely redundant. + while (!this.fileIngestPipelines.isEmpty()) { + pipeline = this.fileIngestPipelines.poll(); + List shutDownErrors = pipeline.shutDown(); + if (!shutDownErrors.isEmpty()) { + logIngestModuleErrors(shutDownErrors); + } + } + break; + } + } + + logIngestModuleErrors(errors); + return errors; + } + + /** + * Starts the data source level ingest progress bar. + */ + private void startDataSourceIngestProgressBar() { + synchronized (this.dataSourceIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + this.dataSource.getName()); + this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. What remains to be done is to find out whether + // the user wants to cancel only the currently executing + // data source ingest module or the entire ingest job. + DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); + String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); + JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); + if (panel.cancelAllDataSourceIngestModules()) { + DataSourceIngestJob.this.cancel(); + } else { + DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule(); + } + return true; + } + }); + this.dataSourceIngestProgress.start(); + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + + /** + * Starts the file level ingest progress bar. + */ + private void startFileIngestProgressBar() { + synchronized (this.fileIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. + DataSourceIngestJob.this.cancel(); + return true; + } + }); + this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + this.fileIngestProgress.start(); + this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + } + } + + /** + * Checks to see if the ingest tasks for the current stage are completed and + * does a stage transition if they are. + */ + private void checkForStageCompleted() { + synchronized (this.stageCompletionCheckLock) { + if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) { + switch (this.stage) { + case FIRST: + this.finishFirstStage(); + break; + case SECOND: + this.finish(); + break; + } + } + } + } + + /** + * Shuts down the first stage ingest pipelines and progress bars and starts + * the second stage, if appropriate. + */ + private void finishFirstStage() { + // Shut down the file ingest pipelines. Note that no shut down is + // required for the data source ingest pipeline because data source + // ingest modules do not have a shutdown() method. + List errors = new ArrayList<>(); + while (!this.fileIngestPipelines.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelines.poll(); + errors.addAll(pipeline.shutDown()); + } + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + // Finish the first stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + + // Finish the file ingest progress bar, if it hasn't already + // been finished. + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + this.fileIngestProgress.finish(); + this.fileIngestProgress = null; + } + } + + /** + * Start the second stage, if appropriate. + */ + if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } else { + this.finish(); + } + } + + /** + * Shuts down the ingest pipelines and progress bars for this job. + */ + private void finish() { + this.stage = DataSourceIngestJob.Stages.FINALIZATION; + + // Finish the second stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + + this.parentJob.dataSourceJobFinished(this); + } + + /** + * Write ingest module errors to the log. + * + * @param errors The errors. + */ + private void logIngestModuleErrors(List errors) { + for (IngestModuleError error : errors) { + DataSourceIngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS + } + } + + /** + * Gets the currently running data source level ingest module. + * + * @return The currently running module, may be null. + */ + DataSourceIngestPipeline.DataSourceIngestModuleDecorator getCurrentDataSourceIngestModule() { + if (null != this.currentDataSourceIngestPipeline) { + return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); + } else { + return null; + } + } + + /** + * Requests a temporary cancellation of data source level ingest in order to + * stop the currently executing data source ingest module. + */ + private void cancelCurrentDataSourceIngestModule() { + this.currentDataSourceIngestModuleCancelled = true; + } + + /** + * Gets a snapshot of this jobs state and performance. + * + * @return An ingest job statistics object. + */ + IngestJobSnapshot getSnapshot() { + return new IngestJobSnapshot(); + } + + /** + * Stores basic diagnostic statistics for an ingest job. + */ + class IngestJobSnapshot { + + private final long jobId; + private final String dataSource; + private final long startTime; + private final long processedFiles; + private final long estimatedFilesToProcess; + private final long snapShotTime; + private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; + + /** + * Constructs an object to store basic diagnostic statistics for an + * ingest job. + */ + IngestJobSnapshot() { + this.jobId = DataSourceIngestJob.this.id; + this.dataSource = DataSourceIngestJob.this.dataSource.getName(); + this.startTime = DataSourceIngestJob.this.createTime; + synchronized (DataSourceIngestJob.this.fileIngestProgressLock) { + this.processedFiles = DataSourceIngestJob.this.processedFiles; + this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess; + this.snapShotTime = new Date().getTime(); + } + + /** + * Get a snapshot of the tasks currently in progress for this job. + */ + this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); + } + + /** + * Gets the identifier of the ingest job that is the subject of this + * snapshot. + * + * @return The ingest job id. + */ + long getJobId() { + return this.jobId; + } + + /** + * Gets the name of the data source associated with the ingest job that + * is the subject of this snapshot. + * + * @return A data source name string. + */ + String getDataSource() { + return dataSource; + } + + /** + * Gets files per second throughput since the ingest job that is the + * subject of this snapshot started. + * + * @return Files processed per second (approximate). + */ + double getSpeed() { + return (double) processedFiles / ((snapShotTime - startTime) / 1000); + } + + /** + * Gets the time the ingest job was started. + * + * @return The start time as number of milliseconds since January 1, + * 1970, 00:00:00 GMT. + */ + long getStartTime() { + return startTime; + } + + /** + * Gets time these statistics were collected. + * + * @return The statistics collection time as number of milliseconds + * since January 1, 1970, 00:00:00 GMT. + */ + long getSnapshotTime() { + return snapShotTime; + } + + /** + * Gets the number of files processed for the job so far. + * + * @return The number of processed files. + */ + long getFilesProcessed() { + return processedFiles; + } + + /** + * Gets an estimate of the files that still need to be processed for + * this job. + * + * @return The estimate. + */ + long getFilesEstimated() { + return estimatedFilesToProcess; + } + + long getRootQueueSize() { + return this.tasksSnapshot.getRootQueueSize(); + } + + long getDirQueueSize() { + return this.tasksSnapshot.getDirectoryTasksQueueSize(); + } + + long getFileQueueSize() { + return this.tasksSnapshot.getFileQueueSize(); + } + + long getDsQueueSize() { + return this.tasksSnapshot.getDsQueueSize(); + } + + long getRunningListSize() { + return this.tasksSnapshot.getRunningListSize(); + } + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index bde810e21d..3c0a3361da 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -25,9 +25,9 @@ import org.netbeans.api.progress.ProgressHandle; */ public class DataSourceIngestModuleProgress { - private final IngestJob job; + private final DataSourceIngestJob job; - DataSourceIngestModuleProgress(IngestJob job) { + DataSourceIngestModuleProgress(DataSourceIngestJob job) { this.job = job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 54d06023ee..1ffd7852b0 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -19,37 +19,61 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source ingest modules. It starts them, - * shuts them down, and runs them in sequential order. + * A pipeline consisting of a sequence of data source level ingest modules. The + * pipeline starts the modules, runs them in sequential order, and shuts them + * down. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; + private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); + private volatile DataSourceIngestModuleDecorator currentModule; - DataSourceIngestPipeline(IngestJob job, List moduleTemplates) { + /** + * Constructs a pipeline consisting of a sequence of data source level + * ingest modules. The pipeline starts the modules, runs them in sequential + * order, and shuts them down. + * + * @param job The ingest job to which this pipeline belongs. + * @param moduleTemplates The ingest module templates that define the + * pipeline. + */ + DataSourceIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - // Create an ingest module instance from each data source ingest module - // template. + /** + * Create a data source level ingest module instance from each ingest + * module template. + */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); modules.add(module); } - } + } } + /** + * Indicates whether or not there are any modules in this pipeline. + * + * @return True or false. + */ boolean isEmpty() { return modules.isEmpty(); } + /** + * Starts up the ingest module in this pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ List startUp() { List errors = new ArrayList<>(); for (DataSourceIngestModuleDecorator module : modules) { @@ -62,11 +86,20 @@ final class DataSourceIngestPipeline { return errors; } + /** + * Runs a data source through the ingest modules in sequential order. + * + * @param task A data source level ingest task containing a data source to + * be processed. + * @return A list of ingest module errors, possible empty. + */ List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); for (DataSourceIngestModuleDecorator module : modules) { try { + module.setStartTime(); + this.currentModule = module; String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", module.getDisplayName(), dataSource.getName()); @@ -79,40 +112,97 @@ final class DataSourceIngestPipeline { } if (this.job.isCancelled()) { break; - } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { + } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { this.job.currentDataSourceIngestModuleCancellationCompleted(); } } + this.currentModule = null; ingestManager.setIngestTaskProgressCompleted(task); return errors; } - private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + /** + * Gets the currently running module. + */ + DataSourceIngestModuleDecorator getCurrentlyRunningModule() { + return this.currentModule; + } + + /** + * This class decorates a data source level ingest module with a display + * name and a start time. + */ + static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; + private Date startTime; + /** + * Constructs an object that decorates a data source level ingest module + * with a display name and a running time. + * + * @param module The data source level ingest module to be decorated. + * @param displayName + */ DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; + this.startTime = new Date(); } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { - return module.getClass().getCanonicalName(); + return this.module.getClass().getCanonicalName(); } + /** + * Gets a module name suitable for display in a UI. + * + * @return The display name. + */ String getDisplayName() { - return displayName; + return this.displayName; + } + + /** + * Sets the start time to the current time. + */ + void setStartTime() { + this.startTime = new Date(); + } + + /** + * Gets the time the decorated ingest module started processing the data + * source. + * + * @return The start time. + */ + Date getStartTime() { + return this.startTime; } + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); + this.module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - return module.process(dataSource, statusHelper); + this.startTime = new Date(); + return this.module.process(dataSource, statusHelper); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index 4fecebbe84..66fa744682 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.ingest; final class DataSourceIngestTask extends IngestTask { - DataSourceIngestTask(IngestJob job) { + DataSourceIngestTask(DataSourceIngestJob job) { super(job); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 329c717d5c..0be6d355e9 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -29,10 +29,10 @@ import org.sleuthkit.datamodel.AbstractFile; final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJob job; + private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); - FileIngestPipeline(IngestJob job, List moduleTemplates) { + FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; // Create an ingest module instance from each file ingest module diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 6fa02cf4fc..41a4045b12 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -29,7 +29,7 @@ final class FileIngestTask extends IngestTask { private final AbstractFile file; - FileIngestTask(IngestJob job, AbstractFile file) { + FileIngestTask(DataSourceIngestJob job, AbstractFile file) { super(job); this.file = file; } @@ -53,8 +53,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestJob job = getIngestJob(); - IngestJob otherJob = other.getIngestJob(); + DataSourceIngestJob job = getIngestJob(); + DataSourceIngestJob otherJob = other.getIngestJob(); if (job != otherJob && (job == null || !job.equals(otherJob))) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index ff463321a7..d24121221f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -19,490 +19,133 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.logging.Level; -import javax.swing.JOptionPane; -import org.netbeans.api.progress.ProgressHandle; -import org.netbeans.api.progress.ProgressHandleFactory; -import org.openide.util.Cancellable; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; +import java.util.concurrent.atomic.AtomicLong; import org.sleuthkit.datamodel.Content; /** - * Encapsulates a data source to be processed and the settings, ingest module - * pipelines, and progress bars that are used to process it. + * Runs a collection of data sources through a set of ingest modules specified + * via ingest job settings. */ -final class IngestJob { +public final class IngestJob { - private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); - - /** - * These fields define an ingest job: a unique ID supplied by the ingest - * manager, the user's ingest job settings, and the data source to be - * processed. - */ + private static final AtomicLong nextJobId = new AtomicLong(0L); private final long id; - private final IngestJobSettings ingestJobSettings; - private final Content dataSource; + private final Map dataSourceJobs; + private boolean cancelled; /** - * An ingest job runs in stages. - */ - private static enum Stages { - - /** - * Setting up for processing. - */ - INITIALIZATION, - /** - * Running high priority data source level ingest modules and file level - * ingest modules. - */ - FIRST, - /** - * Running lower priority, usually long-running, data source level - * ingest modules. - */ - SECOND, - /** - * Cleaning up. - */ - FINALIZATION - }; - private Stages stage; - private final Object stageCompletionCheckLock; - - /** - * An ingest job has separate data source level ingest module pipelines for - * each processing stage. Longer running, lower priority modules belong in - * the second stage pipeline. - */ - private final Object dataSourceIngestPipelineLock; - private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; - private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; - - /** - * An ingest job has a collection of identical file level ingest module - * pipelines, one for each file level ingest thread in the ingest manager. - */ - private final LinkedBlockingQueue fileIngestPipelines; - - /** - * An ingest job supports cancellation of either the currently running data - * source level ingest module or the entire ingest job. - */ - private volatile boolean currentDataSourceIngestModuleCancelled; - private volatile boolean cancelled; - - /** - * An ingest job uses the task scheduler singleton to create and queue the - * ingest tasks that make up the job. - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - - /** - * These fields are used to report data source level ingest progress for the - * ingest job. - */ - private final Object dataSourceIngestProgressLock; - private ProgressHandle dataSourceIngestProgress; - - /** - * These fields are used to report file level ingest task progress for the - * ingest job. - */ - private final Object fileIngestProgressLock; - private final List filesInProgress; - private long estimatedFilesToProcess; - private long processedFiles; - private ProgressHandle fileIngestProgress; - - /** - * This field is used to record the creation of the ingest job. - */ - private final long createTime; - - /** - * Constructs an ingest job. + * Constructs an ingest job that runs a collection of data sources through a + * set of ingest modules specified via ingest job settings. * - * @param id The identifier assigned to the job. - * @param dataSource The data source to be ingested. - * @param settings The settings for the ingest job. + * @param dataSources The data sources to be ingested. + * @param settings The ingest job settings. */ - IngestJob(long id, Content dataSource, IngestJobSettings settings) { - this.id = id; - this.dataSource = dataSource; - this.ingestJobSettings = settings; - this.dataSourceIngestPipelineLock = new Object(); - this.fileIngestPipelines = new LinkedBlockingQueue<>(); - this.filesInProgress = new ArrayList<>(); - this.dataSourceIngestProgressLock = new Object(); - this.fileIngestProgressLock = new Object(); - this.stage = IngestJob.Stages.INITIALIZATION; - this.stageCompletionCheckLock = new Object(); - this.createTime = new Date().getTime(); + IngestJob(Collection dataSources, IngestJobSettings settings) { + this.id = IngestJob.nextJobId.getAndIncrement(); + this.dataSourceJobs = new HashMap<>(); + for (Content dataSource : dataSources) { + DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings); + this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); + } } - /** - * Gets the identifier assigned to this job. - * - * @return The job identifier. - */ - long getId() { + public long getId() { return this.id; } - /** - * Gets the data source to be ingested by this job. - * - * @return A Content object representing the data source. - */ - Content getDataSource() { - return this.dataSource; - } - - /** - * Gets whether or not unallocated space should be processed as part of this - * job. - * - * @return True or false. - */ - boolean shouldProcessUnallocatedSpace() { - return this.ingestJobSettings.getProcessUnallocatedSpace(); - } - - /** - * Passes the data source for this job through the currently active data - * source level ingest pipeline. - * - * @param task A data source ingest task wrapping the data source. - */ - void process(DataSourceIngestTask task) { - try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - /** - * Run the data source through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - } + synchronized public ProgressSnapshot getProgressSnapshot() { + DataSourceIngestModuleHandle moduleHandle = null; + Date fileAnalysisStartTime = null; + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + DataSourceIngestPipeline.DataSourceIngestModuleDecorator module = dataSourceJob.getCurrentDataSourceIngestModule(); + if (null != module) { + moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - - /** - * Shut down the data source ingest progress bar right away. Data - * source-level processing is finished for this stage. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. - * - * @param task A file ingest task. - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or putting to the file ingest - * pipelines collection. - */ - void process(FileIngestTask task) throws InterruptedException { - try { - if (!this.isCancelled()) { - /** - * Get a file ingest pipeline not currently in use by another - * file ingest thread. - */ - FileIngestPipeline pipeline = this.fileIngestPipelines.take(); - if (!pipeline.isEmpty()) { - /** - * Get the file to process. - */ - AbstractFile file = task.getFile(); - - /** - * Update the file ingest progress bar. - */ - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); - } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); - } - this.filesInProgress.add(file.getName()); - } - - /** - * Run the file through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - /** - * Update the file ingest progress bar again, in case the - * file was being displayed. - */ - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); - } else { - this.fileIngestProgress.progress(""); - } - } - } - } - - /** - * Relinquish the pipeline so it can be reused by another file - * ingest thread. - */ - this.fileIngestPipelines.put(pipeline); - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Adds more files to an ingest job, i.e., extracted or carved files. Not - * currently supported for the second stage of the job. - * - * @param files A list of files to add. - */ - void addFiles(List files) { - /** - * Note: This implementation assumes that this is being called by an an - * ingest module running code on an ingest thread that is holding a - * reference to an ingest task, so no task completion check is done. - */ - if (IngestJob.Stages.FIRST == this.stage) { - for (AbstractFile file : files) { - IngestJob.taskScheduler.scheduleFileIngestTask(this, file); - } - } else { - IngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + // RJCTODO: File analysis time thing } - /** - * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a task completion check would not be necessary. This is a - * bit of defensive programming. - */ - this.checkForStageCompleted(); + return new ProgressSnapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); } - /** - * Updates the display name of the data source level ingest progress bar. - * - * @param displayName The new display name. - */ - void updateDataSourceIngestProgressBarDisplayName(String displayName) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); - } - } + synchronized public void cancelDataSourceIngestModule(DataSourceIngestModuleHandle module) { + DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); + // RJCTODO: check equality, etc. } - /** - * Switches the data source level ingest progress bar to determinate mode. - * This should be called if the total work units to process the data source - * is known. - * - * @param workUnits Total number of work units for the processing of the - * data source. - */ - void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); - } - } + synchronized public void cancel() { + for (DataSourceIngestJob job : this.dataSourceJobs.values()) { + job.cancel(); } - } - - /** - * Switches the data source level ingest progress bar to indeterminate mode. - * This should be called if the total work units to process the data source - * is unknown. - */ - void switchDataSourceIngestProgressBarToIndeterminate() { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with the number of work - * units performed, if in the determinate mode. - * - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); - } - } - } - } - - /** - * Updates the data source level ingest progress with a new task name, where - * the task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - */ - void advanceDataSourceIngestProgressBar(String currentTask) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with a new task name - * and the number of work units performed, if in the determinate mode. The - * task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); - } - } - } - - /** - * Queries whether or not a temporary cancellation of data source level - * ingest in order to stop the currently executing data source level ingest - * module is in effect. - * - * @return True or false. - */ - boolean currentDataSourceIngestModuleIsCancelled() { - return this.currentDataSourceIngestModuleCancelled; - } - - /** - * Rescind a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module. - */ - void currentDataSourceIngestModuleCancellationCompleted() { - this.currentDataSourceIngestModuleCancelled = false; - - /** - * A new progress bar must be created because the cancel button of the - * previously constructed component is disabled by NetBeans when the - * user selects the "OK" button of the cancellation confirmation dialog - * popped up by NetBeans when the progress bar cancel button was - * pressed. - */ - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - this.startDataSourceIngestProgressBar(); - } - } - - /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. - */ - void cancel() { - /** - * Put a cancellation message on data source level ingest progress bar, - * if it is still running. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (dataSourceIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - dataSource.getName()); - dataSourceIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), - "IngestJob.progress.cancelling", - displayName)); - } - } - - /** - * Put a cancellation message on the file level ingest progress bar, if - * it is still running. - */ - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", - displayName)); - } - } - this.cancelled = true; + } - /** - * Tell the task scheduler to cancel all pending tasks, i.e., tasks not - * not being performed by an ingest thread. - */ - IngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); - this.checkForStageCompleted(); + synchronized public boolean isCancelled() { + return this.cancelled; } /** - * Queries whether or not cancellation of ingest i.e., a shutdown of the - * data source level and file level ingest pipelines, has been requested. - * - * @return True or false. + * A thread-safe snapshot of ingest job progress. */ - boolean isCancelled() { - return this.cancelled; + public static final class ProgressSnapshot { + + private final DataSourceIngestModuleHandle dataSourceModule; + private final Date fileAnalysisStartTime; + private final boolean cancelled; + + private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { + this.dataSourceModule = dataSourceModule; + this.fileAnalysisStartTime = fileAnalysisStartTime; + this.cancelled = cancelled; + } + + public DataSourceIngestModuleHandle runningDataSourceIngestModule() { + /** + * It is safe to hand out this reference because the object is + * immutable and the mutable + * DataSourceIngestPipeline.DataSourceIngestModuleDecorator is + * hidden from public access. + */ + return this.dataSourceModule; + } + + public boolean fileAnalysisIsRunning() { + return (null != this.fileAnalysisStartTime); + } + + public Date fileAnalysisStartTime() { + return new Date(this.fileAnalysisStartTime.getTime()); + } + + public boolean isCancelled() { + return this.cancelled; + } + + } + + public static class DataSourceIngestModuleHandle { + + private final long dataSourceIngestJobId; + private final DataSourceIngestPipeline.DataSourceIngestModuleDecorator module; + + private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.DataSourceIngestModuleDecorator module) { + this.dataSourceIngestJobId = dataSourceIngestJobId; + this.module = module; + } + + public String displayName() { + return this.module.getDisplayName(); + } + + public Date startTime() { + // RJCTODO + return new Date(); + } + } /** @@ -510,536 +153,56 @@ final class IngestJob { * * @return A collection of ingest module startup errors, empty on success. */ - List start(List ingestModuleTemplates) { - this.createIngestPipelines(ingestModuleTemplates); - List errors = startUpIngestPipelines(); - if (errors.isEmpty()) { - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - this.startFirstStage(); - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } + List start() { + boolean hasIngestPipeline = false; + List errors = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + errors.addAll(dataSourceJob.start()); + hasIngestPipeline = dataSourceJob.hasIngestPipeline(); } return errors; } - /** - * Creates the file and data source ingest pipelines. - * - * @param ingestModuleTemplates Ingest module templates to use to populate - * the pipelines. - */ - private void createIngestPipelines(List ingestModuleTemplates) { - /** - * Make mappings of ingest module factory class names to templates. - */ - Map dataSourceModuleTemplates = new HashMap<>(); - Map fileModuleTemplates = new HashMap<>(); - for (IngestModuleTemplate template : ingestModuleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } - if (template.isFileIngestModuleTemplate()) { - fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); - } - } - - /** - * Use the mappings and the ingest pipelines configuration to create - * ordered lists of ingest module templates for each ingest pipeline. - */ - IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); - - /** - * Add any module templates that were not specified in the pipelines - * configuration to an appropriate pipeline - either the first stage - * data source ingest pipeline or the file ingest pipeline. - */ - for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { - firstStageDataSourceModuleTemplates.add(template); - } - for (IngestModuleTemplate template : fileModuleTemplates.values()) { - fileIngestModuleTemplates.add(template); - } - - /** - * Construct the data source ingest pipelines. - */ - this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - - /** - * Construct the file ingest pipelines, one per file ingest thread. - */ - try { - int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); - } - } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full queue. - * Blocking should actually never happen here, but reset the - * interrupted flag rather than just swallowing the exception. - */ - Thread.currentThread().interrupt(); - } - } - - /** - * Use an ordered list of ingest module factory class names to create an - * ordered output list of ingest module templates for an ingest pipeline. - * The ingest module templates are removed from the input collection as they - * are added to the output collection. - * - * @param ingestModuleTemplates A mapping of ingest module factory class - * names to ingest module templates. - * @param pipelineConfig An ordered list of ingest module factory class - * names representing an ingest pipeline. - * @return - */ - private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { - List templates = new ArrayList<>(); - for (String moduleClassName : pipelineConfig) { - if (ingestModuleTemplates.containsKey(moduleClassName)) { - templates.add(ingestModuleTemplates.remove(moduleClassName)); - } - } - return templates; - } - - /** - * Starts the first stage of the job. - */ - private void startFirstStage() { - this.stage = IngestJob.Stages.FIRST; - - /** - * Start one or both of the first stage ingest progress bars. - */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - IngestJob.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - IngestJob.taskScheduler.scheduleFileIngestTasks(this); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never to check for completion of this stage of the job. - */ - this.checkForStageCompleted(); - } - } - - /** - * Starts the second stage of the ingest job. - */ - private void startSecondStage() { - this.stage = IngestJob.Stages.SECOND; - this.startDataSourceIngestProgressBar(); - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; - } - IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } - /** * Checks to see if this job has at least one ingest pipeline. * * @return True or false. */ boolean hasIngestPipeline() { - return this.hasFirstStageDataSourceIngestPipeline() - || this.hasFileIngestPipeline() - || this.hasSecondStageDataSourceIngestPipeline(); - } - - /** - * Checks to see if this job has a first stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasFirstStageDataSourceIngestPipeline() { - return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if this job has a second stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasSecondStageDataSourceIngestPipeline() { - return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if the job has a file level ingest pipeline. - * - * @return True or false. - */ - private boolean hasFileIngestPipeline() { - return (this.fileIngestPipelines.peek().isEmpty() == false); - } - - /** - * Starts up each of the file and data source level ingest modules to - * collect possible errors. - * - * @return A collection of ingest module startup errors, empty on success. - */ - private List startUpIngestPipelines() { - List errors = new ArrayList<>(); - - // Start up the first stage data source ingest pipeline. - errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); - - // Start up the second stage data source ingest pipeline. - errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); - - // Start up the file ingest pipelines (one per file ingest thread). - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { - errors.addAll(pipeline.startUp()); - if (!errors.isEmpty()) { - // If there are start up errors, the ingest job will not proceed - // and the errors will ultimately be reported to the user for - // possible remedy so shut down the pipelines now that an - // attempt has been made to start up the data source ingest - // pipeline and at least one copy of the file ingest pipeline. - // pipeline. There is no need to complete starting up all of the - // file ingest pipeline copies since any additional start up - // errors are likely redundant. - while (!this.fileIngestPipelines.isEmpty()) { - pipeline = this.fileIngestPipelines.poll(); - List shutDownErrors = pipeline.shutDown(); - if (!shutDownErrors.isEmpty()) { - logIngestModuleErrors(shutDownErrors); - } - } + boolean hasIngestPipeline = false; + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + if (dataSourceJob.hasIngestPipeline()) { + hasIngestPipeline = true; break; } } - - logIngestModuleErrors(errors); - return errors; + return hasIngestPipeline; } /** - * Starts the data source level ingest progress bar. - */ - private void startDataSourceIngestProgressBar() { - synchronized (this.dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. What remains to be done is to find out whether - // the user wants to cancel only the currently executing - // data source ingest module or the entire ingest job. - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { - IngestJob.this.cancel(); - } else { - IngestJob.this.cancelCurrentDataSourceIngestModule(); - } - return true; - } - }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - - /** - * Starts the file level ingest progress bar. - */ - private void startFileIngestProgressBar() { - synchronized (this.fileIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. - IngestJob.this.cancel(); - return true; - } - }); - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); - } - } - - /** - * Checks to see if the ingest tasks for the current stage are completed and - * does a stage transition if they are. - */ - private void checkForStageCompleted() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - switch (this.stage) { - case FIRST: - this.finishFirstStage(); - break; - case SECOND: - this.finish(); - break; - } - } - } - } - - /** - * Shuts down the first stage ingest pipelines and progress bars and starts - * the second stage, if appropriate. - */ - private void finishFirstStage() { - // Shut down the file ingest pipelines. Note that no shut down is - // required for the data source ingest pipeline because data source - // ingest modules do not have a shutdown() method. - List errors = new ArrayList<>(); - while (!this.fileIngestPipelines.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelines.poll(); - errors.addAll(pipeline.shutDown()); - } - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; - } - } - - /** - * Start the second stage, if appropriate. - */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } else { - this.finish(); - } - } - - /** - * Shuts down the ingest pipelines and progress bars for this job. - */ - private void finish() { - this.stage = IngestJob.Stages.FINALIZATION; - - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - IngestManager.getInstance().finishJob(this); - } - - /** - * Write ingest module errors to the log. + * Provides a callback for completed data source ingest jobs, allowing the + * ingest job to notify the ingest manager when it is complete. * - * @param errors The errors. + * @param dataSourceIngestJob A completed data source ingest job. */ - private void logIngestModuleErrors(List errors) { - for (IngestModuleError error : errors) { - IngestJob.logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); //NON-NLS + synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { + this.dataSourceJobs.remove(dataSourceIngestJob.getId()); + if (this.dataSourceJobs.isEmpty()) { + IngestManager.getInstance().finishJob(this); } } - + /** - * Requests a temporary cancellation of data source level ingest in order to - * stop the currently executing data source ingest module. - */ - private void cancelCurrentDataSourceIngestModule() { - this.currentDataSourceIngestModuleCancelled = true; - } - - /** - * Gets a snapshot of this jobs state and performance. + * Gets a snapshot of this job's state and performance. * - * @return An ingest job statistics object. + * @return A list of */ - IngestJobSnapshot getSnapshot() { - return new IngestJobSnapshot(); + List getSnapshot() { + List snapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + snapshots.add(dataSourceJob.getSnapshot()); + } + return snapshots; } - - /** - * Stores basic diagnostic statistics for an ingest job. - */ - class IngestJobSnapshot { - - private final long jobId; - private final String dataSource; - private final long startTime; - private final long processedFiles; - private final long estimatedFilesToProcess; - private final long snapShotTime; - private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; - - /** - * Constructs an object to store basic diagnostic statistics for an - * ingest job. - */ - IngestJobSnapshot() { - this.jobId = IngestJob.this.id; - this.dataSource = IngestJob.this.dataSource.getName(); - this.startTime = IngestJob.this.createTime; - synchronized (IngestJob.this.fileIngestProgressLock) { - this.processedFiles = IngestJob.this.processedFiles; - this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; - this.snapShotTime = new Date().getTime(); - } - - /** - * Get a snapshot of the tasks currently in progress for this job. - */ - this.tasksSnapshot = IngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); - } - - /** - * Gets the identifier of the ingest job that is the subject of this - * snapshot. - * - * @return The ingest job id. - */ - long getJobId() { - return this.jobId; - } - - /** - * Gets the name of the data source associated with the ingest job that - * is the subject of this snapshot. - * - * @return A data source name string. - */ - String getDataSource() { - return dataSource; - } - - /** - * Gets files per second throughput since the ingest job that is the - * subject of this snapshot started. - * - * @return Files processed per second (approximate). - */ - double getSpeed() { - return (double) processedFiles / ((snapShotTime - startTime) / 1000); - } - - /** - * Gets the time the ingest job was started. - * - * @return The start time as number of milliseconds since January 1, - * 1970, 00:00:00 GMT. - */ - long getStartTime() { - return startTime; - } - - /** - * Gets time these statistics were collected. - * - * @return The statistics collection time as number of milliseconds - * since January 1, 1970, 00:00:00 GMT. - */ - long getSnapshotTime() { - return snapShotTime; - } - - /** - * Gets the number of files processed for the job so far. - * - * @return The number of processed files. - */ - long getFilesProcessed() { - return processedFiles; - } - - /** - * Gets an estimate of the files that still need to be processed for - * this job. - * - * @return The estimate. - */ - long getFilesEstimated() { - return estimatedFilesToProcess; - } - - long getRootQueueSize() { - return this.tasksSnapshot.getRootQueueSize(); - } - - long getDirQueueSize() { - return this.tasksSnapshot.getDirectoryTasksQueueSize(); - } - - long getFileQueueSize() { - return this.tasksSnapshot.getFileQueueSize(); - } - - long getDsQueueSize() { - return this.tasksSnapshot.getDsQueueSize(); - } - - long getRunningListSize() { - return this.tasksSnapshot.getRunningListSize(); - } - - } - + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 6587a20d19..020c05812b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -28,9 +28,9 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJobContext { - private final IngestJob ingestJob; + private final DataSourceIngestJob ingestJob; - IngestJobContext(IngestJob ingestJob) { + IngestJobContext(DataSourceIngestJob ingestJob) { this.ingestJob = ingestJob; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 7d32adbcd3..624a9fa2ac 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -56,10 +56,8 @@ public class IngestManager { private static IngestManager instance = null; /** - * The ingest manager assigns a unique ID to each ingest job and maintains a - * mapping of job IDs to jobs. + * The ingest manager maintains a mapping of ingest job IDs to ingest jobs. */ - private final AtomicLong nextJobId = new AtomicLong(0L); private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); /** @@ -67,8 +65,8 @@ public class IngestManager { * pools is given a unique thread/task ID. */ // TODO: It is no longer necessary to have multiple thread pools. - private final AtomicLong nextThreadId = new AtomicLong(0L); - + private final AtomicLong nextThreadId = new AtomicLong(0L); + /** * Ingest jobs are started on a pool thread by ingest job starters. A * mapping of thread/task IDs to the result objects associated with each @@ -216,13 +214,18 @@ public class IngestManager { /** * Starts an ingest job, i.e., processing by ingest modules, for each data - * source in a collection of data sources. + * source in a collection of data sources. Note that if the provide UI + * argument is set to true, it is assumed this method is being called on the + * EDT and a worker thread will be dispatched to start the job. * * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param doMessageBoxes Whether or not to display message boxes for errors. + * @param provideUI Whether or not to support user interaction, e.g., + * showing message boxes and reporting progress through the NetBeans + * Progress API. + * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doMessageBoxes) { + public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean provideUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -232,7 +235,7 @@ public class IngestManager { } long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, doMessageBoxes)); + Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, provideUI)); ingestJobStarters.put(taskId, task); } @@ -406,19 +409,19 @@ public class IngestManager { } /** - * Starts an ingest job for a data source. + * Starts an ingest job for a collection of data sources. * - * @param dataSource The data source to ingest. + * @param dataSource The data sources to ingest. * @param settings The settings for the job. * @return A collection of ingest module start up errors, empty on success. */ - private List startJob(Content dataSource, IngestJobSettings settings) { + private List startJob(Collection dataSources, IngestJobSettings settings) { List errors = new ArrayList<>(); if (this.jobCreationIsEnabled) { - long jobId = this.nextJobId.incrementAndGet(); - IngestJob job = new IngestJob(jobId, dataSource, settings); + IngestJob job = new IngestJob(dataSources, settings); + long jobId = job.getId(); this.jobsById.put(jobId, job); - errors = job.start(settings.getEnabledIngestModuleTemplates()); + errors = job.start(); if (errors.isEmpty() && job.hasIngestPipeline()) { this.fireIngestJobStarted(jobId); IngestManager.logger.log(Level.INFO, "Ingest job {0} started", jobId); @@ -628,10 +631,10 @@ public class IngestManager { * * @return A list of ingest job state snapshots. */ - List getIngestJobSnapshots() { - List snapShots = new ArrayList<>(); + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); for (IngestJob job : this.jobsById.values()) { - snapShots.add(job.getSnapshot()); + snapShots.addAll(job.getSnapshot()); } return snapShots; } @@ -684,61 +687,35 @@ public class IngestManager { return true; } }); - progress.start(dataSources.size()); + progress.start(); /** - * Try to start the ingest jobs. + * Try to start the ingest job. */ - int workUnitsCompleted = 0; - for (Content dataSource : this.dataSources) { - - /** - * Cancellation check. - */ - if (Thread.currentThread().isInterrupted()) { - return null; + List errors = IngestManager.this.startJob(this.dataSources, this.settings); + if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); } - - /** - * Add a "subtitle" to the display name of the progress bar - * to indicate an ingest job is being started for this data - * source. - */ - String progressMessage = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.progressingDisplayName", - dataSource.getName()); - progress.progress(progressMessage); - - /** - * Start an ingest job for this data source. - */ - List errors = IngestManager.this.startJob(dataSource, this.settings); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); - } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - } - - progress.progress(progressMessage, ++workUnitsCompleted); + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); } return null; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 47042c8335..7514a597c0 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -163,7 +163,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final String[] columnNames = {"Job ID", "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; - private List jobSnapshots; + private List jobSnapshots; private IngestJobTableModel() { refresh(); @@ -191,7 +191,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { - IngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); + DataSourceIngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); Object cellValue; switch (columnIndex) { case 0: diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 7ff6634f64..cfef33be7a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -23,15 +23,15 @@ import org.sleuthkit.datamodel.Content; abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; - private final IngestJob job; + private final DataSourceIngestJob job; private long threadId; - IngestTask(IngestJob job) { + IngestTask(DataSourceIngestJob job) { this.job = job; threadId = NOT_SET; } - IngestJob getIngestJob() { + DataSourceIngestJob getIngestJob() { return job; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index d401b1439c..2adfb3ca40 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -148,7 +148,7 @@ final class IngestTasksScheduler { * @throws InterruptedException if the calling thread is blocked due to a * full tasks queue and is interrupted. */ - synchronized void scheduleIngestTasks(IngestJob job) { + synchronized void scheduleIngestTasks(DataSourceIngestJob job) { // Scheduling of both a data source ingest task and file ingest tasks // for a job must be an atomic operation. Otherwise, the data source // task might be completed before the file tasks are scheduled, @@ -163,7 +163,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleDataSourceIngestTask(IngestJob job) { + synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) { DataSourceIngestTask task = new DataSourceIngestTask(job); this.tasksInProgress.add(task); try { @@ -183,7 +183,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to be scheduled. */ - synchronized void scheduleFileIngestTasks(IngestJob job) { + synchronized void scheduleFileIngestTasks(DataSourceIngestJob job) { // Get the top level files for the data source associated with this job // and add them to the root directories priority queue. List topLevelFiles = getTopLevelFiles(job.getDataSource()); @@ -203,7 +203,7 @@ final class IngestTasksScheduler { * @param job The job for which the tasks are to be scheduled. * @param file The file to be associated with the task. */ - synchronized void scheduleFileIngestTask(IngestJob job, AbstractFile file) { + synchronized void scheduleFileIngestTask(DataSourceIngestJob job, AbstractFile file) { FileIngestTask task = new FileIngestTask(job, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { this.tasksInProgress.add(task); @@ -228,7 +228,7 @@ final class IngestTasksScheduler { * @param job The job for which the query is to be performed. * @return True or false. */ - synchronized boolean tasksForJobAreCompleted(IngestJob job) { + synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) { for (IngestTask task : tasksInProgress) { if (task.getIngestJob().getId() == job.getId()) { return false; @@ -245,7 +245,7 @@ final class IngestTasksScheduler { * * @param job The job for which the tasks are to to canceled. */ - synchronized void cancelPendingTasksForIngestJob(IngestJob job) { + synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { long jobId = job.getId(); this.removeTasksForJob(this.rootDirectoryTasks, jobId); this.removeTasksForJob(this.directoryTasks, jobId); From 3cf575029567f49d9c3e7af5d2efb81dea27b900 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 12 Dec 2014 18:52:38 -0500 Subject: [PATCH 49/84] Continue refactoring and implementation of new INgestJob class --- .../autopsy/ingest/DataSourceIngestJob.java | 17 +++- .../ingest/DataSourceIngestPipeline.java | 24 ++--- .../autopsy/ingest/FileIngestPipeline.java | 83 ++++++++++++++--- .../sleuthkit/autopsy/ingest/IngestJob.java | 90 +++++++++++++------ .../autopsy/ingest/IngestManager.java | 6 +- .../ingest/IngestProgressSnapshotPanel.java | 4 +- 6 files changed, 167 insertions(+), 57 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 254510bb6c..6e1b921757 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -328,6 +328,15 @@ final class DataSourceIngestJob { this.checkForStageCompleted(); } + /** + * Queries whether or not this job is running file ingest. + * + * @return True or false. + */ + boolean fileIngestIsRunning() { + return ((DataSourceIngestJob.Stages.FIRST == this.stage) && this.hasFileIngestPipeline()); + } + /** * Updates the display name of the data source level ingest progress bar. * @@ -932,14 +941,14 @@ final class DataSourceIngestJob { * * @return An ingest job statistics object. */ - IngestJobSnapshot getSnapshot() { - return new IngestJobSnapshot(); + Snapshot getSnapshot() { + return new Snapshot(); } /** * Stores basic diagnostic statistics for an ingest job. */ - class IngestJobSnapshot { + class Snapshot { private final long jobId; private final String dataSource; @@ -953,7 +962,7 @@ final class DataSourceIngestJob { * Constructs an object to store basic diagnostic statistics for an * ingest job. */ - IngestJobSnapshot() { + Snapshot() { this.jobId = DataSourceIngestJob.this.id; this.dataSource = DataSourceIngestJob.this.dataSource.getName(); this.startTime = DataSourceIngestJob.this.createTime; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 1ffd7852b0..734effc789 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -25,9 +25,9 @@ import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * A pipeline consisting of a sequence of data source level ingest modules. The - * pipeline starts the modules, runs them in sequential order, and shuts them - * down. + * This class manages a sequence of data source level ingest modules. It starts + * the modules, runs data sources through them, and shuts them down when data + * source level ingest is complete. */ final class DataSourceIngestPipeline { @@ -37,9 +37,9 @@ final class DataSourceIngestPipeline { private volatile DataSourceIngestModuleDecorator currentModule; /** - * Constructs a pipeline consisting of a sequence of data source level - * ingest modules. The pipeline starts the modules, runs them in sequential - * order, and shuts them down. + * Constructs an object that manages a sequence of data source level ingest + * modules. It starts the modules, runs data sources through them, and shuts + * them down when data source level ingest is complete. * * @param job The ingest job to which this pipeline belongs. * @param moduleTemplates The ingest module templates that define the @@ -127,7 +127,7 @@ final class DataSourceIngestPipeline { DataSourceIngestModuleDecorator getCurrentlyRunningModule() { return this.currentModule; } - + /** * This class decorates a data source level ingest module with a display * name and a start time. @@ -143,7 +143,7 @@ final class DataSourceIngestPipeline { * with a display name and a running time. * * @param module The data source level ingest module to be decorated. - * @param displayName + * @param displayName The display name. */ DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { this.module = module; @@ -168,14 +168,14 @@ final class DataSourceIngestPipeline { String getDisplayName() { return this.displayName; } - + /** * Sets the start time to the current time. */ void setStartTime() { this.startTime = new Date(); } - + /** * Gets the time the decorated ingest module started processing the data * source. @@ -202,7 +202,7 @@ final class DataSourceIngestPipeline { this.startTime = new Date(); return this.module.process(dataSource, statusHelper); } - + } - + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 0be6d355e9..3bbdd7fa12 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -19,24 +19,38 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * This class manages a sequence of file ingest modules. It starts them, - * shuts them down, and runs a file through them. + * This class manages a sequence of file level ingest modules. It starts the + * modules, runs files through them, and shuts them down when file level ingest + * is complete. */ final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); + private Date startTime; + /** + * Constructs an object that manages a sequence of file level ingest + * modules. It starts the modules, runs files through them, and shuts them + * down when file level ingest is complete. + * + * @param job The ingest job of which this pipeline is a part. + * @param moduleTemplates The ingest module templates that define the + * pipeline. + */ FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - // Create an ingest module instance from each file ingest module - // template. + /** + * Create an ingest module instance from each file ingest module + * template. + */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); @@ -45,12 +59,28 @@ final class FileIngestPipeline { } } + /** + * Returns the time when the pipeline began processing files. + * + * @return The file processing start time, may be null. + */ + Date getProcessingStartTime() { + return this.startTime; + } + + /** + * Queries whether or not the pipeline has been configured with at least one + * file level ingest module. + * + * @return True or false. + */ boolean isEmpty() { return this.modules.isEmpty(); } /** - * Start up all of the modules in the pipeline. + * Start up all of the modules in the pipeline. + * * @return List of errors or empty list if no errors */ List startUp() { @@ -66,13 +96,15 @@ final class FileIngestPipeline { } /** - * Process the file down the pipeline of modules. - * Startup must have been called before this is called. - * - * @param file File to analyze - * @return List of errors or empty list if no errors + * Runs a file through the ingest modules in sequential order. + * + * @param task A file level ingest task containing a file to be processed. + * @return A list of ingest module errors, possible empty. */ List process(FileIngestTask task) { + if (null == this.startTime) { + this.startTime = new Date(); + } List errors = new ArrayList<>(); AbstractFile file = task.getFile(); for (FileIngestModuleDecorator module : this.modules) { @@ -106,37 +138,68 @@ final class FileIngestPipeline { return errors; } + /** + * This class decorates a file level ingest module with a display name. + */ private static final class FileIngestModuleDecorator implements FileIngestModule { private final FileIngestModule module; private final String displayName; + /** + * Constructs an object that decorates a file level ingest module with a + * display name. + * + * @param module The file level ingest module to be decorated. + * @param displayName The display name. + */ FileIngestModuleDecorator(FileIngestModule module, String displayName) { this.module = module; this.displayName = displayName; } + /** + * Gets the class name of the decorated ingest module. + * + * @return The class name. + */ String getClassName() { return module.getClass().getCanonicalName(); } + /** + * Gets a module name suitable for display in a UI. + * + * @return The display name. + */ String getDisplayName() { return displayName; } + /** + * @inheritDoc + */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { module.startUp(context); } + /** + * @inheritDoc + */ @Override public IngestModule.ProcessResult process(AbstractFile file) { return module.process(file); } + /** + * @inheritDoc + */ @Override public void shutDown() { module.shutDown(); } + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index d24121221f..c7731aecb5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -33,7 +33,7 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJob { - private static final AtomicLong nextJobId = new AtomicLong(0L); + private static final AtomicLong nextId = new AtomicLong(0L); private final long id; private final Map dataSourceJobs; private boolean cancelled; @@ -46,7 +46,7 @@ public final class IngestJob { * @param settings The ingest job settings. */ IngestJob(Collection dataSources, IngestJobSettings settings) { - this.id = IngestJob.nextJobId.getAndIncrement(); + this.id = IngestJob.nextId.getAndIncrement(); this.dataSourceJobs = new HashMap<>(); for (Content dataSource : dataSources) { DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings); @@ -54,36 +54,82 @@ public final class IngestJob { } } + /** + * Gets the unique identifier assigned to this ingest job. + * + * @return The job identifier. + */ public long getId() { return this.id; } - synchronized public ProgressSnapshot getProgressSnapshot() { + /** + * Gets a snapshot of the state and performance of this ingest job. + * + * @return The snapshot. + */ + synchronized public Snapshot getSnapshot() { + // TODO: There are race conditions here that can be easily fixed by + // eliminating the child jobs per plan. DataSourceIngestModuleHandle moduleHandle = null; Date fileAnalysisStartTime = null; for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + /** + * Only one data source ingest module should be running at a time, + * so grab the first one. There is a race condition here. + */ DataSourceIngestPipeline.DataSourceIngestModuleDecorator module = dataSourceJob.getCurrentDataSourceIngestModule(); if (null != module) { moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - // RJCTODO: File analysis time thing + + } - return new ProgressSnapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); + return new Snapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); } - synchronized public void cancelDataSourceIngestModule(DataSourceIngestModuleHandle module) { + /** + * Gets snapshots of the state and performance of this ingest job's child + * data source ingest jobs. + * + * @return A list of data source ingest job snapshots. + */ + synchronized List getDetailedSnapshot() { + List snapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + snapshots.add(dataSourceJob.getSnapshot()); + } + return snapshots; + } + + /** + * + * @param module + */ + synchronized public void cancelIngestModule(DataSourceIngestModuleHandle module) { DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); // RJCTODO: check equality, etc. } + /** + * Requests cancellation of this ingest job, i.e., a shutdown of its data + * source level and file level ingest pipelines. + */ synchronized public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { job.cancel(); } - this.cancelled = true; + this.cancelled = true; // RJCTODO: make children push cancellation up } + /** + * Queries whether or not cancellation of this ingest job, i.e., a shutdown + * of its data source level and file level ingest pipelines, has been + * requested. + * + * @return True or false. + */ synchronized public boolean isCancelled() { return this.cancelled; } @@ -91,13 +137,13 @@ public final class IngestJob { /** * A thread-safe snapshot of ingest job progress. */ - public static final class ProgressSnapshot { + public static final class Snapshot { private final DataSourceIngestModuleHandle dataSourceModule; private final Date fileAnalysisStartTime; private final boolean cancelled; - private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { + private Snapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { this.dataSourceModule = dataSourceModule; this.fileAnalysisStartTime = fileAnalysisStartTime; this.cancelled = cancelled; @@ -127,6 +173,11 @@ public final class IngestJob { } + /** + * A handle to a data source level ingest module that can be used to get + * basic information about the module and to request cancellation, i.e., + * shut down, of the module. + */ public static class DataSourceIngestModuleHandle { private final long dataSourceIngestJobId; @@ -149,9 +200,9 @@ public final class IngestJob { } /** - * Starts up the ingest pipelines and ingest progress bars. + * Starts up the ingest pipelines and ingest progress bars for this job. * - * @return A collection of ingest module startup errors, empty on success. + * @return A collection of ingest module start up errors, empty on success. */ List start() { boolean hasIngestPipeline = false; @@ -164,7 +215,7 @@ public final class IngestJob { } /** - * Checks to see if this job has at least one ingest pipeline. + * Checks to see if this ingest job has at least one ingest pipeline. * * @return True or false. */ @@ -191,18 +242,5 @@ public final class IngestJob { IngestManager.getInstance().finishJob(this); } } - - /** - * Gets a snapshot of this job's state and performance. - * - * @return A list of - */ - List getSnapshot() { - List snapshots = new ArrayList<>(); - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - snapshots.add(dataSourceJob.getSnapshot()); - } - return snapshots; - } - + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 624a9fa2ac..0602579188 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -631,10 +631,10 @@ public class IngestManager { * * @return A list of ingest job state snapshots. */ - List getIngestJobSnapshots() { - List snapShots = new ArrayList<>(); + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); for (IngestJob job : this.jobsById.values()) { - snapShots.addAll(job.getSnapshot()); + snapShots.addAll(job.getDetailedSnapshot()); } return snapShots; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 7514a597c0..b21b1dd3df 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -163,7 +163,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final String[] columnNames = {"Job ID", "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"}; - private List jobSnapshots; + private List jobSnapshots; private IngestJobTableModel() { refresh(); @@ -191,7 +191,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { - DataSourceIngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex); + DataSourceIngestJob.Snapshot snapShot = jobSnapshots.get(rowIndex); Object cellValue; switch (columnIndex) { case 0: From 1e2d7df467229228cbf8829f24dc76d08dd42db4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sat, 13 Dec 2014 20:40:24 -0500 Subject: [PATCH 50/84] Remove EDT queueing of operations within Case class --- .../sleuthkit/autopsy/casemodule/Case.java | 78 ++++++++----------- 1 file changed, 31 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index ba6360fa00..cb409ad7c7 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.casemodule; -import java.awt.EventQueue; import java.awt.Frame; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; @@ -1078,7 +1077,7 @@ public class Case implements SleuthkitCase.ErrorObserver { } //case change helper - private static void doCaseChange(final Case toChangeTo) { + private static void doCaseChange(Case toChangeTo) { logger.log(Level.INFO, "Changing Case to: " + toChangeTo); //NON-NLS if (toChangeTo != null) { // new case is open @@ -1086,49 +1085,39 @@ public class Case implements SleuthkitCase.ErrorObserver { Case.clearTempFolder(); checkSubFolders(toChangeTo); - EventQueue.invokeLater(new Runnable() { - @Override - public void run() { - // enable these menus - CallableSystemAction.get(AddImageAction.class).setEnabled(true); - CallableSystemAction.get(CaseCloseAction.class).setEnabled(true); - CallableSystemAction.get(CasePropertiesAction.class).setEnabled(true); - CallableSystemAction.get(CaseDeleteAction.class).setEnabled(true); // Delete Case menu - - if (toChangeTo.hasData()) { - // open all top components - CoreComponentControl.openCoreWindows(); - } else { - // close all top components - CoreComponentControl.closeCoreWindows(); - } - } - }); + // enable these menus + CallableSystemAction.get(AddImageAction.class).setEnabled(true); + CallableSystemAction.get(CaseCloseAction.class).setEnabled(true); + CallableSystemAction.get(CasePropertiesAction.class).setEnabled(true); + CallableSystemAction.get(CaseDeleteAction.class).setEnabled(true); // Delete Case menu + if (toChangeTo.hasData()) { + // open all top components + CoreComponentControl.openCoreWindows(); + } else { + // close all top components + CoreComponentControl.closeCoreWindows(); + } } else { // case is closed - EventQueue.invokeLater(new Runnable() { - @Override - public void run() { - // close all top components first - CoreComponentControl.closeCoreWindows(); + // close all top components first + CoreComponentControl.closeCoreWindows(); - // disable these menus - CallableSystemAction.get(AddImageAction.class).setEnabled(false); // Add Image menu - CallableSystemAction.get(CaseCloseAction.class).setEnabled(false); // Case Close menu - CallableSystemAction.get(CasePropertiesAction.class).setEnabled(false); // Case Properties menu - CallableSystemAction.get(CaseDeleteAction.class).setEnabled(false); // Delete Case menu + // disable these menus + CallableSystemAction.get(AddImageAction.class).setEnabled(false); // Add Image menu + CallableSystemAction.get(CaseCloseAction.class).setEnabled(false); // Case Close menu + CallableSystemAction.get(CasePropertiesAction.class).setEnabled(false); // Case Properties menu + CallableSystemAction.get(CaseDeleteAction.class).setEnabled(false); // Delete Case menu - //clear pending notifications - MessageNotifyUtil.Notify.clear(); + //clear pending notifications + MessageNotifyUtil.Notify.clear(); - Frame f = WindowManager.getDefault().getMainWindow(); - f.setTitle(Case.getAppName()); // set the window name to just application name - //try to force gc to happen - System.gc(); - System.gc(); - } - }); + Frame f = WindowManager.getDefault().getMainWindow(); + f.setTitle(Case.getAppName()); // set the window name to just application name + + //try to force gc to happen + System.gc(); + System.gc(); } //log memory usage after case changed @@ -1141,13 +1130,8 @@ public class Case implements SleuthkitCase.ErrorObserver { private static void doCaseNameChange(String newCaseName) { // update case name if (!newCaseName.equals("")) { - EventQueue.invokeLater(new Runnable() { - @Override - public void run() { - Frame f = WindowManager.getDefault().getMainWindow(); - f.setTitle(newCaseName + " - " + Case.getAppName()); // set the window name to the new value - } - }); + Frame f = WindowManager.getDefault().getMainWindow(); + f.setTitle(newCaseName + " - " + Case.getAppName()); // set the window name to the new value } } @@ -1202,4 +1186,4 @@ public class Case implements SleuthkitCase.ErrorObserver { } return hasData; } -} +} \ No newline at end of file From 6c6f5687126840103dd3fad0155f05d28f31dcfa Mon Sep 17 00:00:00 2001 From: Karl Mortensen Date: Mon, 15 Dec 2014 09:04:40 -0500 Subject: [PATCH 51/84] Use final local instead of member variable, add instanceof for JDialog. --- .../autopsy/ingest/IngestProgressSnapshotDialog.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java index 0e59b3ceb5..90ee2b6078 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java @@ -37,8 +37,7 @@ public final class IngestProgressSnapshotDialog extends JDialog { private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestProgressSnapshotDialog.title.text"); private static final Dimension DIMENSIONS = new Dimension(500, 300); - private JDialog pseudoOwner = null; - + /** * Constructs a non-modal instance of the dialog with its own frame. */ @@ -54,8 +53,8 @@ public final class IngestProgressSnapshotDialog extends JDialog { */ public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) { super((Window) owner, TITLE, ModalityType.MODELESS); - if (shouldBeModal) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. - pseudoOwner = (JDialog) owner; + if (shouldBeModal && owner instanceof JDialog) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal. + final JDialog pseudoOwner = (JDialog) owner; addWindowListener(new WindowAdapter() { @Override public void windowClosed(WindowEvent e) { // Put it back to how it was before we manipulated it. From 2de22db810f85fd56b6ac4351cd4cfb80841c695 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Dec 2014 16:09:55 -0500 Subject: [PATCH 52/84] Check in new public IngestJob and IngestManager API stubs --- .../autopsy/ingest/DataSourceIngestJob.java | 2 +- .../ingest/DataSourceIngestPipeline.java | 18 +-- .../autopsy/ingest/FileIngestPipeline.java | 77 +++++++--- .../sleuthkit/autopsy/ingest/IngestJob.java | 131 +++++++++++++----- .../autopsy/ingest/IngestManager.java | 35 +++-- 5 files changed, 179 insertions(+), 84 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 6e1b921757..c244a16f7c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -920,7 +920,7 @@ final class DataSourceIngestJob { * * @return The currently running module, may be null. */ - DataSourceIngestPipeline.DataSourceIngestModuleDecorator getCurrentDataSourceIngestModule() { + DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { if (null != this.currentDataSourceIngestPipeline) { return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); } else { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 734effc789..d4dd78ca34 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -28,13 +28,15 @@ import org.sleuthkit.datamodel.Content; * This class manages a sequence of data source level ingest modules. It starts * the modules, runs data sources through them, and shuts them down when data * source level ingest is complete. + *

    + * This class is not thread-safe. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; - private final List modules = new ArrayList<>(); - private volatile DataSourceIngestModuleDecorator currentModule; + private final List modules = new ArrayList<>(); + private volatile PipelineModule currentModule; /** * Constructs an object that manages a sequence of data source level ingest @@ -54,7 +56,7 @@ final class DataSourceIngestPipeline { */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); modules.add(module); } } @@ -76,7 +78,7 @@ final class DataSourceIngestPipeline { */ List startUp() { List errors = new ArrayList<>(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall @@ -96,7 +98,7 @@ final class DataSourceIngestPipeline { List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.setStartTime(); this.currentModule = module; @@ -124,7 +126,7 @@ final class DataSourceIngestPipeline { /** * Gets the currently running module. */ - DataSourceIngestModuleDecorator getCurrentlyRunningModule() { + PipelineModule getCurrentlyRunningModule() { return this.currentModule; } @@ -132,7 +134,7 @@ final class DataSourceIngestPipeline { * This class decorates a data source level ingest module with a display * name and a start time. */ - static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + static class PipelineModule implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; @@ -145,7 +147,7 @@ final class DataSourceIngestPipeline { * @param module The data source level ingest module to be decorated. * @param displayName The display name. */ - DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { + PipelineModule(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; this.startTime = new Date(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 3bbdd7fa12..3e5b09f3ba 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -26,14 +26,17 @@ import org.sleuthkit.datamodel.AbstractFile; /** * This class manages a sequence of file level ingest modules. It starts the * modules, runs files through them, and shuts them down when file level ingest - * is complete. + * is complete. + *

    + * This class is not thread-safe. */ final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; - private final List modules = new ArrayList<>(); + private final List modules = new ArrayList<>(); private Date startTime; + private boolean running; /** * Constructs an object that manages a sequence of file level ingest @@ -53,24 +56,15 @@ final class FileIngestPipeline { */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { - FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); modules.add(module); } } } /** - * Returns the time when the pipeline began processing files. - * - * @return The file processing start time, may be null. - */ - Date getProcessingStartTime() { - return this.startTime; - } - - /** - * Queries whether or not the pipeline has been configured with at least one - * file level ingest module. + * Queries whether or not this pipeline has been configured with at least + * one file level ingest module. * * @return True or false. */ @@ -79,35 +73,54 @@ final class FileIngestPipeline { } /** - * Start up all of the modules in the pipeline. + * Starts up all of the modules in the pipeline. * - * @return List of errors or empty list if no errors + * @return List of start up errors, possibly empty. */ List startUp() { List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + if (this.running) { + throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS + } + + for (PipelineModule module : this.modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } + this.running = true; return errors; } + /** + * Returns the start up time of the pipeline. + * + * @return The file processing start time, may be null. + */ + Date getStartTime() { + return this.startTime; + } + /** * Runs a file through the ingest modules in sequential order. * * @param task A file level ingest task containing a file to be processed. - * @return A list of ingest module errors, possible empty. + * @return A list of processing errors, possible empty. */ List process(FileIngestTask task) { + if (!this.running) { + throw new IllegalStateException("Attempt to process a file with pipeline that is not running"); //NON-NLS + } + if (null == this.startTime) { this.startTime = new Date(); } + List errors = new ArrayList<>(); AbstractFile file = task.getFile(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(file); @@ -126,9 +139,18 @@ final class FileIngestPipeline { return errors; } + /** + * Shuts down all of the modules in the pipeline. + * + * @return A list of shut down errors, possibly empty. + */ List shutDown() { + if (!this.running) { + throw new IllegalStateException("Attempt to shut down a pipeline that is not running"); //NON-NLS + } + List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { module.shutDown(); } catch (Throwable ex) { // Catch-all exception firewall @@ -138,10 +160,19 @@ final class FileIngestPipeline { return errors; } + /** + * Queries whether or not this file ingest level pipeline is running. + * + * @return True or false. + */ + boolean isRunning() { + return this.running; + } + /** * This class decorates a file level ingest module with a display name. */ - private static final class FileIngestModuleDecorator implements FileIngestModule { + private static final class PipelineModule implements FileIngestModule { private final FileIngestModule module; private final String displayName; @@ -153,7 +184,7 @@ final class FileIngestPipeline { * @param module The file level ingest module to be decorated. * @param displayName The display name. */ - FileIngestModuleDecorator(FileIngestModule module, String displayName) { + PipelineModule(FileIngestModule module, String displayName) { this.module = module; this.displayName = displayName; } @@ -168,7 +199,7 @@ final class FileIngestPipeline { } /** - * Gets a module name suitable for display in a UI. + * Gets display name of the decorated ingest module. * * @return The display name. */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index c7731aecb5..9c51e744d9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -68,32 +68,34 @@ public final class IngestJob { * * @return The snapshot. */ - synchronized public Snapshot getSnapshot() { - // TODO: There are race conditions here that can be easily fixed by - // eliminating the child jobs per plan. + synchronized public ProgressSnapshot getSnapshot() { + /** + * There are race conditions in the code that follows, but they are not + * important because this is just a coarse-grained status report. If + * stale data is returned in any single snapshot, it will be corrected + * in subsequent snapshots. + */ DataSourceIngestModuleHandle moduleHandle = null; - Date fileAnalysisStartTime = null; + boolean fileIngestRunning = false; + Date fileIngestStartTime = null; for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - /** - * Only one data source ingest module should be running at a time, - * so grab the first one. There is a race condition here. - */ - DataSourceIngestPipeline.DataSourceIngestModuleDecorator module = dataSourceJob.getCurrentDataSourceIngestModule(); + DataSourceIngestPipeline.PipelineModule module = dataSourceJob.getCurrentDataSourceIngestModule(); if (null != module) { moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - - + + // RJCTODO: For each data source job, check for a running flag and + // get the oldest start data for the start dates, if any. } - return new Snapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); + return new ProgressSnapshot(moduleHandle, fileIngestRunning, fileIngestStartTime, this.cancelled); } /** * Gets snapshots of the state and performance of this ingest job's child * data source ingest jobs. * - * @return A list of data source ingest job snapshots. + * @return A list of data source ingest job progress snapshots. */ synchronized List getDetailedSnapshot() { List snapshots = new ArrayList<>(); @@ -104,29 +106,34 @@ public final class IngestJob { } /** + * Requests cancellation of a specific data source level ingest module. + * Returns immediately, but there may be a delay before the ingest module + * responds by stopping processing, if it is still running when the request + * is made. * - * @param module + * @param module The handle of the data source ingest module to be canceled, + * which can obtained from a progress snapshot. */ synchronized public void cancelIngestModule(DataSourceIngestModuleHandle module) { DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); - // RJCTODO: check equality, etc. + // RJCTODO: Pass through the cancellation request. } /** - * Requests cancellation of this ingest job, i.e., a shutdown of its data - * source level and file level ingest pipelines. + * Requests cancellation of the data source level and file level ingest + * modules of this ingest job. Returns immediately, but there may be a delay + * before all of the ingest modules respond by stopping processing. */ synchronized public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { job.cancel(); } - this.cancelled = true; // RJCTODO: make children push cancellation up + this.cancelled = true; } /** - * Queries whether or not cancellation of this ingest job, i.e., a shutdown - * of its data source level and file level ingest pipelines, has been - * requested. + * Queries whether or not cancellation of the data source level and file + * level ingest modules of this ingest job has been requested. * * @return True or false. */ @@ -135,38 +142,72 @@ public final class IngestJob { } /** - * A thread-safe snapshot of ingest job progress. + * A snapshot of ingest job progress. */ - public static final class Snapshot { + public static final class ProgressSnapshot { private final DataSourceIngestModuleHandle dataSourceModule; - private final Date fileAnalysisStartTime; + private final boolean fileIngestRunning; + private final Date fileIngestStartTime; private final boolean cancelled; - private Snapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { + /** + * Constructs a snapshot of ingest job progress. + * + * @param dataSourceModule The currently running data source level + * ingest module, may be null + * @param fileIngestRunning Whether or not file ingest is currently + * running. + * @param fileIngestStartTime The start time of file level ingest, may + * be null + * @param cancelled Whether or not a cancellation request has been + * issued. + */ + private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) { this.dataSourceModule = dataSourceModule; - this.fileAnalysisStartTime = fileAnalysisStartTime; + this.fileIngestRunning = fileIngestRunning; + this.fileIngestStartTime = fileIngestStartTime; this.cancelled = cancelled; } + /** + * Gets a handle to the currently running data source level ingest + * module at the time the snapshot is taken. + * + * @return The handle, may be null. + */ public DataSourceIngestModuleHandle runningDataSourceIngestModule() { /** * It is safe to hand out this reference because the object is - * immutable and the mutable - * DataSourceIngestPipeline.DataSourceIngestModuleDecorator is - * hidden from public access. + * immutable. */ return this.dataSourceModule; } - public boolean fileAnalysisIsRunning() { - return (null != this.fileAnalysisStartTime); + /** + * Queries whether or not file level ingest is running at the time the + * snapshot is taken. + * + * @return True or false. + */ + public boolean fileIngestIsRunning() { + return this.fileIngestRunning; } - public Date fileAnalysisStartTime() { - return new Date(this.fileAnalysisStartTime.getTime()); + /** + * Gets the time that file level ingest started. + * + * @return The start time, may be null. + */ + public Date fileIngestStartTime() { + return new Date(this.fileIngestStartTime.getTime()); } + /** + * Queries whether or not a cancellation request has been issued. + * + * @return True or false. + */ public boolean isCancelled() { return this.cancelled; } @@ -181,20 +222,36 @@ public final class IngestJob { public static class DataSourceIngestModuleHandle { private final long dataSourceIngestJobId; - private final DataSourceIngestPipeline.DataSourceIngestModuleDecorator module; + private final DataSourceIngestPipeline.PipelineModule module; - private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.DataSourceIngestModuleDecorator module) { + /** + * Constructs a handle to a data source level ingest module that can be + * used to get basic information about the module and to request + * cancellation of the module. + */ + private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.PipelineModule module) { this.dataSourceIngestJobId = dataSourceIngestJobId; this.module = module; } + /** + * Gets the display name of the data source level ingest module + * associated with this handle. + * + * @return The display name. + */ public String displayName() { return this.module.getDisplayName(); } + /** + * Returns the time the data source level ingest module associated with + * this handle began processing. + * + * @return The module start time. + */ public Date startTime() { - // RJCTODO - return new Date(); + return this.module.getStartTime(); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 0602579188..13d4f4211e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -213,19 +213,15 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for each data - * source in a collection of data sources. Note that if the provide UI - * argument is set to true, it is assumed this method is being called on the - * EDT and a worker thread will be dispatched to start the job. + * Starts an ingest job for a collection of data sources. * * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param provideUI Whether or not to support user interaction, e.g., - * showing message boxes and reporting progress through the NetBeans - * Progress API. + * @param doUI Whether or not to support user interaction, e.g., showing + * message boxes and reporting progress through the NetBeans Progress API. * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean provideUI) { + public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -234,9 +230,17 @@ public class IngestManager { ingestMonitor.start(); } - long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, provideUI)); - ingestJobStarters.put(taskId, task); + if (doUI) { + /** + * Assume request is from code running on the EDT and dispatch to a + * worker thread. + */ + long taskId = nextThreadId.incrementAndGet(); + Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, dataSources, settings, doUI)); + ingestJobStarters.put(taskId, task); + } else { + this.startJob(dataSources, settings); + } } /** @@ -245,6 +249,8 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { + // RJCTODO: This may return the wrong answer if an IngestJobStarter has + // been dispatched to the startIngestJobsThreadPool. return !this.jobsById.isEmpty(); } @@ -640,10 +646,9 @@ public class IngestManager { } /** - * Creates and starts an ingest job, i.e., processing by ingest modules, for - * each data source in a collection of data sources. + * Creates and starts an ingest job for a collection of data sources. */ - private final class IngestJobsStarter implements Callable { + private final class IngestJobStarter implements Callable { private final long threadId; private final Collection dataSources; @@ -651,7 +656,7 @@ public class IngestManager { private final boolean doStartupErrorsMsgBox; private ProgressHandle progress; - IngestJobsStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { + IngestJobStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { this.threadId = threadId; this.dataSources = dataSources; this.settings = settings; From 644a4e8c3881ab6f91cbd07e745921badce2bda8 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Dec 2014 16:14:21 -0500 Subject: [PATCH 53/84] Check in new public IngestJob and IngestManager API stubs --- Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java | 2 +- .../core/core.jar/org/netbeans/core/startup/Bundle.properties | 2 +- .../org/netbeans/core/windows/view/ui/Bundle.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 13d4f4211e..3b791dbfce 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -221,7 +221,7 @@ public class IngestManager { * message boxes and reporting progress through the NetBeans Progress API. * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doUI) { + public synchronized void startIngestJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index b808dd0d96..575a34a2f6 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 17:10:10 -0400 +#Mon, 15 Dec 2014 16:14:04 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index f0109bc92b..9ca23c2a58 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 31 Oct 2014 12:13:36 -0400 +#Mon, 15 Dec 2014 16:14:04 -0500 CTL_MainWindow_Title=Autopsy 3.1.1 CTL_MainWindow_Title_No_Project=Autopsy 3.1.1 From 389e014b2ba6aa4dc35fd590d51c4dce5761f0ca Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Dec 2014 16:17:41 -0500 Subject: [PATCH 54/84] Check in new public IngestJob and IngestManager API stubs --- .../autopsy/casemodule/AddImageWizardIngestConfigPanel.java | 2 +- .../org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java | 4 ++-- .../org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 96208f007f..052d46047c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,7 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel dataSources) { - IngestManager.getInstance().startIngestJobs(dataSources, this.settings, true); + IngestManager.getInstance().startIngestJob(dataSources, this.settings, true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index d8a7536ac1..b69e967ad9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -199,7 +199,7 @@ public final class RunIngestModulesDialog extends JDialog { ingestJobSettings.save(); showWarnings(ingestJobSettings); if (startIngestJob) { - IngestManager.getInstance().startIngestJobs(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); + IngestManager.getInstance().startIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); } setVisible(false); dispose(); From 40bab3b278c431f34c0f823592f8e25df52d0fca Mon Sep 17 00:00:00 2001 From: Kay Bassi Date: Mon, 15 Dec 2014 17:27:41 -0800 Subject: [PATCH 55/84] Checked for non-standard characters. Didn't find any. Re-typed the translation to see if that helps. --- .../autopsy/modules/interestingitems/Bundle_ja.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties index 7660216076..ff6ae984ab 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle_ja.properties @@ -41,7 +41,7 @@ InterestingItemDefsPanel.jLabel3.text=\u30CD\u30FC\u30E0\u30D1\u30BF\u30FC\u30F3 InterestingItemDefsPanel.jLabel4.text=\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\uFF1A InterestingItemDefsPanel.jLabel5.text=\u6982\u8981\uFF1A InterestingItemDefsPanel.jLabel6.text=\u30BB\u30C3\u30C8\u8A73\u7D30 -InterestingItemDefsPanel.jTextArea1.text=\u6307\u5B9A\u3055\u308C\u305F\u6761\u4EF6\u3068\u4E00\u81F4\u3059\u308B\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u7D22\u3059\u308B\u306E\u304C\u53EF\u80FD\u306A\u30E2\u30B8\u30E5\u30FC\u30EB\u3067\u3059\u3002\u5404\u30BB\u30C3\u30C8\u306B\u306F\u30D5\u30A1\u30A4\u30EB\u540D\u304A\u3088\u3073\u30DA\u30A2\u30EC\u30F3\u30C8\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\u3092\u3082\u3068\u306B\u4E00\u81F4\u3059\u308B\u30EB\u30FC\u30EB\u306E\u30EA\u30B9\u30C8\u304C\u3042\u308A\u307E\u3059\u3002 +InterestingItemDefsPanel.jTextArea1.text=\u6307\u5B9A\u3055\u308C\u305F\u6761\u4EF6\u3068\u4E00\u81F4\u3059\u308B\u30D5\u30A1\u30A4\u30EB\u3092\u691C\u7D22\u3059\u308B\u306E\u304C\u53EF\u80FD\u306A\u30E2\u30B8\u30E5\u30FC\u30EB\u3067\u3059\u3002\u5404\u30BB\u30C3\u30C8\u306B\u306F\u30D5\u30A1\u30A4\u30EB\u540D\u304A\u3088\u3073\u30DA\u30A2\u30EC\u30F3\u30C8\u30D1\u30B9\u30D1\u30BF\u30FC\u30F3\u3092\u3082\u3068\u306B\u4E00\u81F4\u3059\u308B\u3001\u30EB\u30FC\u30EB\u30EA\u30B9\u30C8\u304C\u3042\u308A\u307E\u3059\u3002 InterestingItemDefsPanel.newRuleButton.text=\u65B0\u898F\u30EB\u30FC\u30EB InterestingItemDefsPanel.newSetButton.text=\u65B0\u898F\u30BB\u30C3\u30C8 InterestingItemDefsPanel.rulePathFilterRegexCheckBox.text=\u6B63\u898F\u8868\u73FE From d385d9e08585896a8363b255d3505c49aa41f541 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 16 Dec 2014 09:53:12 -0500 Subject: [PATCH 56/84] Update public API of ingest package --- .../autopsy/ingest/DataSourceIngestJob.java | 74 ++++++++++++++----- .../ingest/DataSourceIngestPipeline.java | 23 ++++-- .../autopsy/ingest/FileIngestPipeline.java | 53 +++++++------ .../sleuthkit/autopsy/ingest/IngestJob.java | 19 +++-- .../autopsy/ingest/IngestManager.java | 9 ++- 5 files changed, 115 insertions(+), 63 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index c244a16f7c..9997eb38df 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -93,9 +93,13 @@ final class DataSourceIngestJob { /** * An ingest job has a collection of identical file level ingest module - * pipelines, one for each file level ingest thread in the ingest manager. + * pipelines, one for each file level ingest thread in the ingest manager. A + * blocking queue is used to dole out the pipelines to the threads and an + * ordinary list is used when the ingest job needs to access the pipelines + * to query their status. */ - private final LinkedBlockingQueue fileIngestPipelines; + private final LinkedBlockingQueue fileIngestPipelinesForThreads; + private final List fileIngestPipelines; /** * An ingest job supports cancellation of either the currently running data @@ -110,6 +114,8 @@ final class DataSourceIngestJob { */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + private final boolean doUI; + /** * These fields are used to report data source level ingest progress for the * ingest job. @@ -139,14 +145,17 @@ final class DataSourceIngestJob { * part. * @param dataSource The data source to be ingested. * @param settings The settings for the ingest job. + * @param doUI Whether or not to display progress bars. */ - DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings) { + DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean doUI) { this.parentJob = parentJob; this.id = DataSourceIngestJob.nextJobId.getAndIncrement(); this.dataSource = dataSource; this.settings = settings; + this.doUI = doUI; this.dataSourceIngestPipelineLock = new Object(); - this.fileIngestPipelines = new LinkedBlockingQueue<>(); + this.fileIngestPipelinesForThreads = new LinkedBlockingQueue<>(); + this.fileIngestPipelines = new ArrayList<>(); this.filesInProgress = new ArrayList<>(); this.dataSourceIngestProgressLock = new Object(); this.fileIngestProgressLock = new Object(); @@ -156,7 +165,7 @@ final class DataSourceIngestJob { } /** - * Gets the identifier assigned to this job. + * Gets the unique identifier of this job. * * @return The job identifier. */ @@ -174,8 +183,8 @@ final class DataSourceIngestJob { } /** - * Gets whether or not unallocated space should be processed as part of this - * job. + * Indicates whether or not unallocated space should be processed as part of + * this job. * * @return True or false. */ @@ -239,7 +248,7 @@ final class DataSourceIngestJob { * Get a file ingest pipeline not currently in use by another * file ingest thread. */ - FileIngestPipeline pipeline = this.fileIngestPipelines.take(); + FileIngestPipeline pipeline = this.fileIngestPipelinesForThreads.take(); if (!pipeline.isEmpty()) { /** * Get the file to process. @@ -288,7 +297,7 @@ final class DataSourceIngestJob { * Relinquish the pipeline so it can be reused by another file * ingest thread. */ - this.fileIngestPipelines.put(pipeline); + this.fileIngestPipelinesForThreads.put(pipeline); } } finally { /** @@ -336,7 +345,7 @@ final class DataSourceIngestJob { boolean fileIngestIsRunning() { return ((DataSourceIngestJob.Stages.FIRST == this.stage) && this.hasFileIngestPipeline()); } - + /** * Updates the display name of the data source level ingest progress bar. * @@ -591,7 +600,9 @@ final class DataSourceIngestJob { try { int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); for (int i = 0; i < numberOfFileIngestThreads; ++i) { - this.fileIngestPipelines.put(new FileIngestPipeline(this, fileIngestModuleTemplates)); + FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); + this.fileIngestPipelinesForThreads.put(pipeline); + this.fileIngestPipelines.add(pipeline); } } catch (InterruptedException ex) { /** @@ -664,7 +675,7 @@ final class DataSourceIngestJob { * it is possible, if unlikely, that no file ingest tasks were * actually scheduled since there are files that get filtered out by * the tasks scheduler. In this special case, an ingest thread will - * never to check for completion of this stage of the job. + * never get to check for completion of this stage of the job. */ this.checkForStageCompleted(); } @@ -719,7 +730,8 @@ final class DataSourceIngestJob { * @return True or false. */ private boolean hasFileIngestPipeline() { - return (this.fileIngestPipelines.peek().isEmpty() == false); + // RJCTODO: Consider going to the list instead...what if the pipelines are all checked out by threads? + return (this.fileIngestPipelinesForThreads.peek().isEmpty() == false); } /** @@ -738,7 +750,7 @@ final class DataSourceIngestJob { errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); // Start up the file ingest pipelines (one per file ingest thread). - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { + for (FileIngestPipeline pipeline : this.fileIngestPipelinesForThreads) { errors.addAll(pipeline.startUp()); if (!errors.isEmpty()) { // If there are start up errors, the ingest job will not proceed @@ -749,13 +761,14 @@ final class DataSourceIngestJob { // pipeline. There is no need to complete starting up all of the // file ingest pipeline copies since any additional start up // errors are likely redundant. - while (!this.fileIngestPipelines.isEmpty()) { - pipeline = this.fileIngestPipelines.poll(); + while (!this.fileIngestPipelinesForThreads.isEmpty()) { + pipeline = this.fileIngestPipelinesForThreads.poll(); List shutDownErrors = pipeline.shutDown(); if (!shutDownErrors.isEmpty()) { logIngestModuleErrors(shutDownErrors); } } + this.fileIngestPipelines.clear(); break; } } @@ -850,13 +863,14 @@ final class DataSourceIngestJob { // required for the data source ingest pipeline because data source // ingest modules do not have a shutdown() method. List errors = new ArrayList<>(); - while (!this.fileIngestPipelines.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelines.poll(); + while (!this.fileIngestPipelinesForThreads.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelinesForThreads.poll(); errors.addAll(pipeline.shutDown()); } if (!errors.isEmpty()) { logIngestModuleErrors(errors); } + this.fileIngestPipelines.clear(); // Finish the first stage data source ingest progress bar, if it hasn't // already been finished. @@ -953,6 +967,9 @@ final class DataSourceIngestJob { private final long jobId; private final String dataSource; private final long startTime; + private final DataSourceIngestPipeline.PipelineModule dataSourceLevelModule; + private final boolean fileIngestRunning; + private final Date fileIngestStartTime; private final long processedFiles; private final long estimatedFilesToProcess; private final long snapShotTime; @@ -966,6 +983,27 @@ final class DataSourceIngestJob { this.jobId = DataSourceIngestJob.this.id; this.dataSource = DataSourceIngestJob.this.dataSource.getName(); this.startTime = DataSourceIngestJob.this.createTime; + + /** + * Get a snapshot of data source level ingest. + */ + this.dataSourceLevelModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule(); + + /** + * Get a snapshot of file level ingest. + */ + boolean fileIngestIsRunning = false; + Date fileIngestStart = null; + for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) { + if (pipeline.isRunning()) { + fileIngestIsRunning = true; + if (null == fileIngestStart || pipeline.getStartTime().before(fileIngestStart)) { + fileIngestStart = pipeline.getStartTime(); + } + } + } + this.fileIngestRunning = fileIngestIsRunning; + this.fileIngestStartTime = fileIngestStart; synchronized (DataSourceIngestJob.this.fileIngestProgressLock) { this.processedFiles = DataSourceIngestJob.this.processedFiles; this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index d4dd78ca34..6dcc3ce0bc 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -25,9 +25,9 @@ import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source level ingest modules. It starts - * the modules, runs data sources through them, and shuts them down when data - * source level ingest is complete. + * This class manages a sequence of data source level ingest modules for a data + * source ingest job. It starts the modules, runs data sources through them, and + * shuts them down when data source level ingest is complete. *

    * This class is not thread-safe. */ @@ -37,13 +37,14 @@ final class DataSourceIngestPipeline { private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); private volatile PipelineModule currentModule; + private boolean running; /** * Constructs an object that manages a sequence of data source level ingest * modules. It starts the modules, runs data sources through them, and shuts * them down when data source level ingest is complete. * - * @param job The ingest job to which this pipeline belongs. + * @param job The data source ingest job to which this pipeline belongs. * @param moduleTemplates The ingest module templates that define the * pipeline. */ @@ -63,7 +64,7 @@ final class DataSourceIngestPipeline { } /** - * Indicates whether or not there are any modules in this pipeline. + * Indicates whether or not there are any ingest modules in this pipeline. * * @return True or false. */ @@ -77,6 +78,10 @@ final class DataSourceIngestPipeline { * @return A list of ingest module startup errors, possibly empty. */ List startUp() { + if (this.running) { + throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS + } + List errors = new ArrayList<>(); for (PipelineModule module : modules) { try { @@ -93,9 +98,13 @@ final class DataSourceIngestPipeline { * * @param task A data source level ingest task containing a data source to * be processed. - * @return A list of ingest module errors, possible empty. + * @return A list of processing errors, possible empty. */ List process(DataSourceIngestTask task) { + if (!this.running) { + throw new IllegalStateException("Attempt to process with pipeline that is not running"); //NON-NLS + } + List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); for (PipelineModule module : modules) { @@ -125,6 +134,8 @@ final class DataSourceIngestPipeline { /** * Gets the currently running module. + * + * @return The module, possibly null. */ PipelineModule getCurrentlyRunningModule() { return this.currentModule; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 3e5b09f3ba..f322913690 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -24,11 +24,13 @@ import java.util.List; import org.sleuthkit.datamodel.AbstractFile; /** - * This class manages a sequence of file level ingest modules. It starts the - * modules, runs files through them, and shuts them down when file level ingest - * is complete. + * This class manages a sequence of file level ingest modules for a data source + * ingest job. It starts the modules, runs files through them, and shuts them + * down when file level ingest is complete. *

    - * This class is not thread-safe. + * This class is not thread-safe, it is intended to be used by one file ingest + * thread at at time. However, the running flag is volatile since it may be read + * by another thread looking for a progress snapshot. */ final class FileIngestPipeline { @@ -36,14 +38,14 @@ final class FileIngestPipeline { private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); private Date startTime; - private boolean running; + private volatile boolean running; /** * Constructs an object that manages a sequence of file level ingest * modules. It starts the modules, runs files through them, and shuts them * down when file level ingest is complete. * - * @param job The ingest job of which this pipeline is a part. + * @param job The data source ingest job that owns the pipeline. * @param moduleTemplates The ingest module templates that define the * pipeline. */ @@ -52,7 +54,7 @@ final class FileIngestPipeline { /** * Create an ingest module instance from each file ingest module - * template. + * template and add it to the pipeline. */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { @@ -63,8 +65,7 @@ final class FileIngestPipeline { } /** - * Queries whether or not this pipeline has been configured with at least - * one file level ingest module. + * Indicates whether or not there are any ingest modules in this pipeline. * * @return True or false. */ @@ -73,16 +74,27 @@ final class FileIngestPipeline { } /** - * Starts up all of the modules in the pipeline. + * Queries whether or not this file ingest level pipeline is running. + * + * @return True or false. + */ + boolean isRunning() { + return this.running; + } + + /** + * Starts up all of the ingest modules in the pipeline. * * @return List of start up errors, possibly empty. */ List startUp() { - List errors = new ArrayList<>(); if (this.running) { throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS } - + this.running = true; + this.startTime = new Date(); + + List errors = new ArrayList<>(); for (PipelineModule module : this.modules) { try { module.startUp(new IngestJobContext(this.job)); @@ -90,7 +102,6 @@ final class FileIngestPipeline { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } - this.running = true; return errors; } @@ -111,11 +122,7 @@ final class FileIngestPipeline { */ List process(FileIngestTask task) { if (!this.running) { - throw new IllegalStateException("Attempt to process a file with pipeline that is not running"); //NON-NLS - } - - if (null == this.startTime) { - this.startTime = new Date(); + throw new IllegalStateException("Attempt to process file with pipeline that is not running"); //NON-NLS } List errors = new ArrayList<>(); @@ -157,18 +164,10 @@ final class FileIngestPipeline { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } + this.running = false; return errors; } - /** - * Queries whether or not this file ingest level pipeline is running. - * - * @return True or false. - */ - boolean isRunning() { - return this.running; - } - /** * This class decorates a file level ingest module with a display name. */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 9c51e744d9..01ebfb0765 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -44,12 +44,13 @@ public final class IngestJob { * * @param dataSources The data sources to be ingested. * @param settings The ingest job settings. + * @param doUI Whether or not to do UI interactions. */ - IngestJob(Collection dataSources, IngestJobSettings settings) { + IngestJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { this.id = IngestJob.nextId.getAndIncrement(); this.dataSourceJobs = new HashMap<>(); for (Content dataSource : dataSources) { - DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings); + DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, doUI); this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); } } @@ -97,7 +98,7 @@ public final class IngestJob { * * @return A list of data source ingest job progress snapshots. */ - synchronized List getDetailedSnapshot() { + synchronized List getDetailedSnapshot() { // RJCTODO: Consider renaming List snapshots = new ArrayList<>(); for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { snapshots.add(dataSourceJob.getSnapshot()); @@ -114,11 +115,9 @@ public final class IngestJob { * @param module The handle of the data source ingest module to be canceled, * which can obtained from a progress snapshot. */ - synchronized public void cancelIngestModule(DataSourceIngestModuleHandle module) { - DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); - // RJCTODO: Pass through the cancellation request. - } - + +// RJCTODO + /** * Requests cancellation of the data source level and file level ingest * modules of this ingest job. Returns immediately, but there may be a delay @@ -253,6 +252,10 @@ public final class IngestJob { public Date startTime() { return this.module.getStartTime(); } + + public void cancel() { + // RJCTODO: + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3b791dbfce..d1c56e3967 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -239,7 +239,7 @@ public class IngestManager { Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, dataSources, settings, doUI)); ingestJobStarters.put(taskId, task); } else { - this.startJob(dataSources, settings); + this.startJob(dataSources, settings, doUI); } } @@ -419,12 +419,13 @@ public class IngestManager { * * @param dataSource The data sources to ingest. * @param settings The settings for the job. + * @param doUI Whether or not to interact with the UI * @return A collection of ingest module start up errors, empty on success. */ - private List startJob(Collection dataSources, IngestJobSettings settings) { + private List startJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { List errors = new ArrayList<>(); if (this.jobCreationIsEnabled) { - IngestJob job = new IngestJob(dataSources, settings); + IngestJob job = new IngestJob(dataSources, settings, doUI); long jobId = job.getId(); this.jobsById.put(jobId, job); errors = job.start(); @@ -697,7 +698,7 @@ public class IngestManager { /** * Try to start the ingest job. */ - List errors = IngestManager.this.startJob(this.dataSources, this.settings); + List errors = IngestManager.this.startJob(this.dataSources, this.settings, true); if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { StringBuilder moduleStartUpErrors = new StringBuilder(); for (IngestModuleError error : errors) { From c11b5a0fe070b77ce59b0ff8001ece362b692b42 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 16 Dec 2014 23:19:19 -0500 Subject: [PATCH 57/84] Continue reworking of ingest API --- .../AddImageWizardIngestConfigPanel.java | 2 +- .../autopsy/ingest/DataSourceIngestJob.java | 1269 +++++++++-------- .../ingest/DataSourceIngestPipeline.java | 52 +- .../autopsy/ingest/FileIngestPipeline.java | 42 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 219 +-- .../autopsy/ingest/IngestJobConfigurator.java | 2 +- .../autopsy/ingest/IngestManager.java | 505 ++++--- .../autopsy/ingest/IngestModule.java | 6 + .../ingest/IngestProgressSnapshotPanel.java | 2 +- .../ingest/RunIngestModulesDialog.java | 2 +- 10 files changed, 1055 insertions(+), 1046 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 052d46047c..7ac97c1ca2 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,7 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel fileIngestPipelinesForThreads; - private final List fileIngestPipelines; + private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); + private final List fileIngestPipelines = new ArrayList<>(); /** - * An ingest job supports cancellation of either the currently running data - * source level ingest module or the entire ingest job. + * A data source ingest job supports cancellation of either the currently + * running data source level ingest module or the entire ingest job. + * + * TODO: The currentDataSourceIngestModuleCancelled field and all of the + * code concerned with it is a hack to avoid an API change. The next time an + * API change is legal, a cancel() method needs to be added to the + * IngestModule interface and this field should be removed. The "ingest job + * is canceled" queries should also be removed from the IngestJobContext + * class. */ private volatile boolean currentDataSourceIngestModuleCancelled; private volatile boolean cancelled; /** - * An ingest job uses the task scheduler singleton to create and queue the - * ingest tasks that make up the job. + * A data source ingest job uses the task scheduler singleton to create and + * queue the ingest tasks that make up the job. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - private final boolean doUI; + /** + * A data source ingest job can run interactively using NetBeans progress + * handles. + */ + private final boolean runInteractively; /** - * These fields are used to report data source level ingest progress for the - * ingest job. + * A data source ingest job uses these fields to report data source level + * ingest progress. */ - private final Object dataSourceIngestProgressLock; + private final Object dataSourceIngestProgressLock = new Object(); private ProgressHandle dataSourceIngestProgress; /** - * These fields are used to report file level ingest task progress for the - * ingest job. + * A data source ingest job uses these fields to report file level ingest + * progress. */ - private final Object fileIngestProgressLock; - private final List filesInProgress; + private final Object fileIngestProgressLock = new Object(); + private final List filesInProgress = new ArrayList<>(); private long estimatedFilesToProcess; private long processedFiles; private ProgressHandle fileIngestProgress; /** - * This field is used to record the creation of the ingest job. + * A data source ingest job uses this field to report its creation time. */ private final long createTime; /** - * Constructs an ingest job. + * Constructs an object that encapsulates a data source and the ingest + * module pipelines used to process it. * * @param parentJob The ingest job of which this data source ingest job is a * part. * @param dataSource The data source to be ingested. * @param settings The settings for the ingest job. - * @param doUI Whether or not to display progress bars. + * @param runInteractively Whether or not this job should use NetBeans + * progress handles. */ - DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean doUI) { + DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) { this.parentJob = parentJob; this.id = DataSourceIngestJob.nextJobId.getAndIncrement(); this.dataSource = dataSource; this.settings = settings; - this.doUI = doUI; - this.dataSourceIngestPipelineLock = new Object(); - this.fileIngestPipelinesForThreads = new LinkedBlockingQueue<>(); - this.fileIngestPipelines = new ArrayList<>(); - this.filesInProgress = new ArrayList<>(); - this.dataSourceIngestProgressLock = new Object(); - this.fileIngestProgressLock = new Object(); - this.stage = DataSourceIngestJob.Stages.INITIALIZATION; - this.stageCompletionCheckLock = new Object(); + this.runInteractively = runInteractively; this.createTime = new Date().getTime(); - } - - /** - * Gets the unique identifier of this job. - * - * @return The job identifier. - */ - long getId() { - return this.id; - } - - /** - * Gets the data source to be ingested by this job. - * - * @return A Content object representing the data source. - */ - Content getDataSource() { - return this.dataSource; - } - - /** - * Indicates whether or not unallocated space should be processed as part of - * this job. - * - * @return True or false. - */ - boolean shouldProcessUnallocatedSpace() { - return this.settings.getProcessUnallocatedSpace(); - } - - /** - * Passes the data source for this job through the currently active data - * source level ingest pipeline. - * - * @param task A data source ingest task wrapping the data source. - */ - void process(DataSourceIngestTask task) { - try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - /** - * Run the data source through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - } - } - - /** - * Shut down the data source ingest progress bar right away. Data - * source-level processing is finished for this stage. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. - * - * @param task A file ingest task. - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or putting to the file ingest - * pipelines collection. - */ - void process(FileIngestTask task) throws InterruptedException { - try { - if (!this.isCancelled()) { - /** - * Get a file ingest pipeline not currently in use by another - * file ingest thread. - */ - FileIngestPipeline pipeline = this.fileIngestPipelinesForThreads.take(); - if (!pipeline.isEmpty()) { - /** - * Get the file to process. - */ - AbstractFile file = task.getFile(); - - /** - * Update the file ingest progress bar. - */ - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); - } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); - } - this.filesInProgress.add(file.getName()); - } - - /** - * Run the file through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - /** - * Update the file ingest progress bar again, in case the - * file was being displayed. - */ - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); - } else { - this.fileIngestProgress.progress(""); - } - } - } - } - - /** - * Relinquish the pipeline so it can be reused by another file - * ingest thread. - */ - this.fileIngestPipelinesForThreads.put(pipeline); - } - } finally { - /** - * No matter what happens, do ingest task bookkeeping. - */ - DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Adds more files to an ingest job, i.e., extracted or carved files. Not - * currently supported for the second stage of the job. - * - * @param files A list of files to add. - */ - void addFiles(List files) { - /** - * Note: This implementation assumes that this is being called by an an - * ingest module running code on an ingest thread that is holding a - * reference to an ingest task, so no task completion check is done. - */ - if (DataSourceIngestJob.Stages.FIRST == this.stage) { - for (AbstractFile file : files) { - DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); - } - } else { - DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS - } - - /** - * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a task completion check would not be necessary. This is a - * bit of defensive programming. - */ - this.checkForStageCompleted(); - } - - /** - * Queries whether or not this job is running file ingest. - * - * @return True or false. - */ - boolean fileIngestIsRunning() { - return ((DataSourceIngestJob.Stages.FIRST == this.stage) && this.hasFileIngestPipeline()); - } - - /** - * Updates the display name of the data source level ingest progress bar. - * - * @param displayName The new display name. - */ - void updateDataSourceIngestProgressBarDisplayName(String displayName) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); - } - } - } - - /** - * Switches the data source level ingest progress bar to determinate mode. - * This should be called if the total work units to process the data source - * is known. - * - * @param workUnits Total number of work units for the processing of the - * data source. - */ - void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); - } - } - } - } - - /** - * Switches the data source level ingest progress bar to indeterminate mode. - * This should be called if the total work units to process the data source - * is unknown. - */ - void switchDataSourceIngestProgressBarToIndeterminate() { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with the number of work - * units performed, if in the determinate mode. - * - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(int workUnits) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); - } - } - } - } - - /** - * Updates the data source level ingest progress with a new task name, where - * the task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - */ - void advanceDataSourceIngestProgressBar(String currentTask) { - if (!this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); - } - } - } - } - - /** - * Updates the data source level ingest progress bar with a new task name - * and the number of work units performed, if in the determinate mode. The - * task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { - if (!this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); - } - } - } - - /** - * Queries whether or not a temporary cancellation of data source level - * ingest in order to stop the currently executing data source level ingest - * module is in effect. - * - * @return True or false. - */ - boolean currentDataSourceIngestModuleIsCancelled() { - return this.currentDataSourceIngestModuleCancelled; - } - - /** - * Rescind a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module. - */ - void currentDataSourceIngestModuleCancellationCompleted() { - this.currentDataSourceIngestModuleCancelled = false; - - /** - * A new progress bar must be created because the cancel button of the - * previously constructed component is disabled by NetBeans when the - * user selects the "OK" button of the cancellation confirmation dialog - * popped up by NetBeans when the progress bar cancel button was - * pressed. - */ - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - this.startDataSourceIngestProgressBar(); - } - } - - /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. - */ - void cancel() { - /** - * Put a cancellation message on data source level ingest progress bar, - * if it is still running. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (dataSourceIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - dataSource.getName()); - dataSourceIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), - "IngestJob.progress.cancelling", - displayName)); - } - } - - /** - * Put a cancellation message on the file level ingest progress bar, if - * it is still running. - */ - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", - displayName)); - } - } - - this.cancelled = true; - - /** - * Tell the task scheduler to cancel all pending tasks, i.e., tasks not - * not being performed by an ingest thread. - */ - DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); - this.checkForStageCompleted(); - } - - /** - * Queries whether or not cancellation of ingest i.e., a shutdown of the - * data source level and file level ingest pipelines, has been requested. - * - * @return True or false. - */ - boolean isCancelled() { - return this.cancelled; - } - - /** - * Starts up the ingest pipelines and ingest progress bars. - * - * @return A collection of ingest module startup errors, empty on success. - */ - List start() { - this.createIngestPipelines(settings.getEnabledIngestModuleTemplates()); - List errors = startUpIngestPipelines(); - if (errors.isEmpty()) { - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - this.startFirstStage(); - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } - } - return errors; + this.createIngestPipelines(); } /** * Creates the file and data source ingest pipelines. - * - * @param ingestModuleTemplates Ingest module templates to use to populate - * the pipelines. */ - private void createIngestPipelines(List ingestModuleTemplates) { + private void createIngestPipelines() { + List ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates(); + /** * Make mappings of ingest module factory class names to templates. */ @@ -601,7 +226,7 @@ final class DataSourceIngestJob { int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); for (int i = 0; i < numberOfFileIngestThreads; ++i) { FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); - this.fileIngestPipelinesForThreads.put(pipeline); + this.fileIngestPipelinesQueue.put(pipeline); this.fileIngestPipelines.add(pipeline); } } catch (InterruptedException ex) { @@ -615,16 +240,18 @@ final class DataSourceIngestJob { } /** - * Use an ordered list of ingest module factory class names to create an - * ordered output list of ingest module templates for an ingest pipeline. - * The ingest module templates are removed from the input collection as they - * are added to the output collection. + * Uses an input collection of ingest module templates and a pipeline + * configuration, i.e., an ordered list of ingest module factory class + * names, to create an ordered output list of ingest module templates for an + * ingest pipeline. The ingest module templates are removed from the input + * collection as they are added to the output collection. * * @param ingestModuleTemplates A mapping of ingest module factory class * names to ingest module templates. * @param pipelineConfig An ordered list of ingest module factory class * names representing an ingest pipeline. - * @return + * @return An ordered list of ingest module templates, i.e., an + * uninstantiated pipeline. */ private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { List templates = new ArrayList<>(); @@ -637,60 +264,31 @@ final class DataSourceIngestJob { } /** - * Starts the first stage of the job. + * Gets the identifier of this job. + * + * @return The job identifier. */ - private void startFirstStage() { - this.stage = DataSourceIngestJob.Stages.FIRST; - - /** - * Start one or both of the first stage ingest progress bars. - */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never get to check for completion of this stage of the job. - */ - this.checkForStageCompleted(); - } + long getId() { + return this.id; } /** - * Starts the second stage of the ingest job. + * Gets the data source to be ingested by this job. + * + * @return A Content object representing the data source. */ - private void startSecondStage() { - this.stage = DataSourceIngestJob.Stages.SECOND; - this.startDataSourceIngestProgressBar(); - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; - } - DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + Content getDataSource() { + return this.dataSource; + } + + /** + * Queries whether or not unallocated space should be processed as part of + * this job. + * + * @return True or false. + */ + boolean shouldProcessUnallocatedSpace() { + return this.settings.getProcessUnallocatedSpace(); } /** @@ -725,18 +323,37 @@ final class DataSourceIngestJob { } /** - * Checks to see if the job has a file level ingest pipeline. + * Checks to see if this job has a file level ingest pipeline. * * @return True or false. */ private boolean hasFileIngestPipeline() { - // RJCTODO: Consider going to the list instead...what if the pipelines are all checked out by threads? - return (this.fileIngestPipelinesForThreads.peek().isEmpty() == false); + if (!this.fileIngestPipelines.isEmpty()) { + return !this.fileIngestPipelines.get(0).isEmpty(); + } + return false; } /** - * Starts up each of the file and data source level ingest modules to - * collect possible errors. + * Starts up the ingest pipelines for this job. + * + * @return A collection of ingest module startup errors, empty on success. + */ + List start() { + List errors = startUpIngestPipelines(); + if (errors.isEmpty()) { + if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { + this.startFirstStage(); + } else if (this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } + } + return errors; + } + + /** + * Starts up each of the ingest pipelines for this job to collect any file + * and data source level ingest modules errors that might occur. * * @return A collection of ingest module startup errors, empty on success. */ @@ -750,7 +367,7 @@ final class DataSourceIngestJob { errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); // Start up the file ingest pipelines (one per file ingest thread). - for (FileIngestPipeline pipeline : this.fileIngestPipelinesForThreads) { + for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) { errors.addAll(pipeline.startUp()); if (!errors.isEmpty()) { // If there are start up errors, the ingest job will not proceed @@ -761,14 +378,13 @@ final class DataSourceIngestJob { // pipeline. There is no need to complete starting up all of the // file ingest pipeline copies since any additional start up // errors are likely redundant. - while (!this.fileIngestPipelinesForThreads.isEmpty()) { - pipeline = this.fileIngestPipelinesForThreads.poll(); + while (!this.fileIngestPipelinesQueue.isEmpty()) { + pipeline = this.fileIngestPipelinesQueue.poll(); List shutDownErrors = pipeline.shutDown(); if (!shutDownErrors.isEmpty()) { logIngestModuleErrors(shutDownErrors); } } - this.fileIngestPipelines.clear(); break; } } @@ -778,66 +394,135 @@ final class DataSourceIngestJob { } /** - * Starts the data source level ingest progress bar. + * Starts the first stage of this job. + */ + private void startFirstStage() { + this.stage = DataSourceIngestJob.Stages.FIRST; + + if (this.hasFileIngestPipeline()) { + this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + } + + if (this.runInteractively) { + /** + * Start one or both of the first stage ingest progress bars. + */ + if (this.hasFirstStageDataSourceIngestPipeline()) { + this.startDataSourceIngestProgressBar(); + } + if (this.hasFileIngestPipeline()) { + this.startFileIngestProgressBar(); + } + } + + /** + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + } + + /** + * Schedule the first stage tasks. + */ + if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); + } else if (this.hasFirstStageDataSourceIngestPipeline()) { + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this); + + /** + * No data source ingest task has been scheduled for this stage, and + * it is possible, if unlikely, that no file ingest tasks were + * actually scheduled since there are files that get filtered out by + * the tasks scheduler. In this special case, an ingest thread will + * never get to check for completion of this stage of the job, so do + * it now. + */ + this.checkForStageCompleted(); + } + } + + /** + * Starts the second stage of this ingest job. + */ + private void startSecondStage() { + this.stage = DataSourceIngestJob.Stages.SECOND; + if (this.runInteractively) { + this.startDataSourceIngestProgressBar(); + } + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + } + DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); + } + + /** + * Starts a data source level ingest progress bar for this job. */ private void startDataSourceIngestProgressBar() { - synchronized (this.dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. What remains to be done is to find out whether - // the user wants to cancel only the currently executing - // data source ingest module or the entire ingest job. - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { - DataSourceIngestJob.this.cancel(); - } else { - DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule(); + if (this.runInteractively) { + synchronized (this.dataSourceIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + this.dataSource.getName()); + this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. What remains to be done is to find out whether + // the user wants to cancel only the currently executing + // data source ingest module or the entire ingest job. + DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); + String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); + JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); + if (panel.cancelAllDataSourceIngestModules()) { + DataSourceIngestJob.this.cancel(); + } else { + DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule(); + } + return true; } - return true; - } - }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); + }); + this.dataSourceIngestProgress.start(); + this.dataSourceIngestProgress.switchToIndeterminate(); + } } } /** - * Starts the file level ingest progress bar. + * Starts the file level ingest progress bar for this job. */ private void startFileIngestProgressBar() { - synchronized (this.fileIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. - DataSourceIngestJob.this.cancel(); - return true; - } - }); - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + if (this.runInteractively) { + synchronized (this.fileIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + // If this method is called, the user has already pressed + // the cancel button on the progress bar and the OK button + // of a cancelation confirmation dialog supplied by + // NetBeans. + DataSourceIngestJob.this.cancel(); + return true; + } + }); + this.fileIngestProgress.start(); + this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + } } } /** - * Checks to see if the ingest tasks for the current stage are completed and - * does a stage transition if they are. + * Checks to see if the ingest tasks for the current stage of this job are + * completed and does a stage transition if they are. */ private void checkForStageCompleted() { synchronized (this.stageCompletionCheckLock) { @@ -855,38 +540,39 @@ final class DataSourceIngestJob { } /** - * Shuts down the first stage ingest pipelines and progress bars and starts - * the second stage, if appropriate. + * Shuts down the first stage ingest pipelines and progress bars for this + * job and starts the second stage, if appropriate. */ private void finishFirstStage() { // Shut down the file ingest pipelines. Note that no shut down is // required for the data source ingest pipeline because data source // ingest modules do not have a shutdown() method. List errors = new ArrayList<>(); - while (!this.fileIngestPipelinesForThreads.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelinesForThreads.poll(); + while (!this.fileIngestPipelinesQueue.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); errors.addAll(pipeline.shutDown()); } if (!errors.isEmpty()) { logIngestModuleErrors(errors); } - this.fileIngestPipelines.clear(); - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; + if (this.runInteractively) { + // Finish the first stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } } - } - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; + // Finish the file ingest progress bar, if it hasn't already + // been finished. + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + this.fileIngestProgress.finish(); + this.fileIngestProgress = null; + } } } @@ -906,18 +592,353 @@ final class DataSourceIngestJob { private void finish() { this.stage = DataSourceIngestJob.Stages.FINALIZATION; - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; + if (this.runInteractively) { + // Finish the second stage data source ingest progress bar, if it hasn't + // already been finished. + synchronized (this.dataSourceIngestProgressLock) { + if (this.dataSourceIngestProgress != null) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } } } this.parentJob.dataSourceJobFinished(this); } + /** + * Passes the data source for this job through the currently active data + * source level ingest pipeline. + * + * @param task A data source ingest task wrapping the data source. + */ + void process(DataSourceIngestTask task) { + try { + synchronized (this.dataSourceIngestPipelineLock) { + if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + List errors = new ArrayList<>(); + errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } + + if (this.runInteractively) { + /** + * Shut down the data source ingest progress bar right away. + * Data source-level processing is finished for this stage. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + } + } + } + + } finally { + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Passes a file from the data source for this job through the file level + * ingest pipeline. + * + * @param task A file ingest task. + * @throws InterruptedException if the thread executing this code is + * interrupted while blocked on taking from or putting to the file ingest + * pipelines collection. + */ + void process(FileIngestTask task) throws InterruptedException { + try { + if (!this.isCancelled()) { + FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); + if (!pipeline.isEmpty()) { + AbstractFile file = task.getFile(); + ++this.processedFiles; + + if (this.runInteractively) { + /** + * Update the file ingest progress bar. + */ + synchronized (this.fileIngestProgressLock) { + if (this.processedFiles <= this.estimatedFilesToProcess) { + this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); + } else { + this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); + } + this.filesInProgress.add(file.getName()); + } + } + + /** + * Run the file through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(pipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + + if (this.runInteractively && !this.cancelled) { + synchronized (this.fileIngestProgressLock) { + /** + * Update the file ingest progress bar again, in + * case the file was being displayed. + */ + this.filesInProgress.remove(file.getName()); + if (this.filesInProgress.size() > 0) { + this.fileIngestProgress.progress(this.filesInProgress.get(0)); + } else { + this.fileIngestProgress.progress(""); + } + } + } + } + this.fileIngestPipelinesQueue.put(pipeline); + } + } finally { + DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); + this.checkForStageCompleted(); + } + } + + /** + * Adds more files from the data source for this job to the job, i.e., adds + * extracted or carved files. Not currently supported for the second stage + * of the job. + * + * @param files A list of the files to add. + */ + void addFiles(List files) { + if (DataSourceIngestJob.Stages.FIRST == this.stage) { // RJCTODO: Is this synchronized correctly + for (AbstractFile file : files) { + DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); + } + } else { + DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS + } + + /** + * The intended clients of this method are ingest modules running code + * on an ingest thread that is holding a reference to an ingest task, in + * which case a completion check would not be necessary, so this is a + * bit of defensive programming. + */ + this.checkForStageCompleted(); + } + + /** + * Updates the display name shown on the current data source level ingest + * progress bar for this job. + * + * @param displayName The new display name. + */ + void updateDataSourceIngestProgressBarDisplayName(String displayName) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.setDisplayName(displayName); + } + } + } + + /** + * Switches the data source level ingest progress bar for this job to + * determinate mode. This should be called if the total work units to + * process the data source is known. + * + * @param workUnits Total number of work units for the processing of the + * data source. + */ + void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToDeterminate(workUnits); + } + } + } + } + + /** + * Switches the data source level ingest progress bar for this job to + * indeterminate mode. This should be called if the total work units to + * process the data source is unknown. + */ + void switchDataSourceIngestProgressBarToIndeterminate() { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.switchToIndeterminate(); + } + } + } + } + + /** + * Updates the data source level ingest progress bar for this job with the + * number of work units performed, if in the determinate mode. + * + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(int workUnits) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress("", workUnits); + } + } + } + } + + /** + * Updates the data source level ingest progress for this job with a new + * task name, where the task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + */ + void advanceDataSourceIngestProgressBar(String currentTask) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.dataSourceIngestProgressLock) { + if (null != this.dataSourceIngestProgress) { + this.dataSourceIngestProgress.progress(currentTask); + } + } + } + } + + /** + * Updates the data source level ingest progress bar for this with a new + * task name and the number of work units performed, if in the determinate + * mode. The task name is the "subtitle" under the display name. + * + * @param currentTask The task name. + * @param workUnits Number of work units performed. + */ + void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { + if (this.runInteractively && !this.cancelled) { + synchronized (this.fileIngestProgressLock) { + this.dataSourceIngestProgress.progress(currentTask, workUnits); + } + } + } + + /** + * Queries whether or not a temporary cancellation of data source level + * ingest in order to stop the currently executing data source level ingest + * module is in effect for this job. + * + * @return True or false. + */ + boolean currentDataSourceIngestModuleIsCancelled() { + return this.currentDataSourceIngestModuleCancelled; + } + + /** + * Rescind a temporary cancellation of data source level ingest that was + * used to stop a single data source level ingest module fro this job. + */ + void currentDataSourceIngestModuleCancellationCompleted() { + this.currentDataSourceIngestModuleCancelled = false; + + if (this.runInteractively) { + /** + * A new progress bar must be created because the cancel button of + * the previously constructed component is disabled by NetBeans when + * the user selects the "OK" button of the cancellation confirmation + * dialog popped up by NetBeans when the progress bar cancel button + * is pressed. + */ + synchronized (this.dataSourceIngestProgressLock) { + this.dataSourceIngestProgress.finish(); + this.dataSourceIngestProgress = null; + this.startDataSourceIngestProgressBar(); + } + } + } + + /** + * Gets the currently running data source level ingest module for this job. + * + * @return The currently running module, may be null. + */ + DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { + if (null != this.currentDataSourceIngestPipeline) { + return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); + } else { + return null; + } + } + + /** + * Requests a temporary cancellation of data source level ingest for this + * job in order to stop the currently executing data source ingest module. + */ + void cancelCurrentDataSourceIngestModule() { + this.currentDataSourceIngestModuleCancelled = true; + } + + /** + * Requests cancellation of ingest, i.e., a shutdown of the data source + * level and file level ingest pipelines. + */ + void cancel() { + if (this.runInteractively) { + /** + * Put a cancellation message on data source level ingest progress + * bar, if it is still running. + */ + synchronized (this.dataSourceIngestProgressLock) { + if (dataSourceIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.initialDisplayName", + dataSource.getName()); + dataSourceIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), + "IngestJob.progress.cancelling", + displayName)); + } + } + + /** + * Put a cancellation message on the file level ingest progress bar, + * if it is still running. + */ + synchronized (this.fileIngestProgressLock) { + if (this.fileIngestProgress != null) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + this.dataSource.getName()); + this.fileIngestProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", + displayName)); + } + } + } + + this.cancelled = true; + + /** + * Tell the task scheduler to cancel all pending tasks, i.e., tasks not + * not being performed by an ingest thread. + */ + DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); + this.checkForStageCompleted(); + } + + /** + * Queries whether or not cancellation, i.e., a shutdown of the data source + * level and file level ingest pipelines for this job, has been requested. + * + * @return True or false. + */ + boolean isCancelled() { + return this.cancelled; + } + /** * Write ingest module errors to the log. * @@ -929,27 +950,6 @@ final class DataSourceIngestJob { } } - /** - * Gets the currently running data source level ingest module. - * - * @return The currently running module, may be null. - */ - DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { - if (null != this.currentDataSourceIngestPipeline) { - return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); - } else { - return null; - } - } - - /** - * Requests a temporary cancellation of data source level ingest in order to - * stop the currently executing data source ingest module. - */ - private void cancelCurrentDataSourceIngestModule() { - this.currentDataSourceIngestModuleCancelled = true; - } - /** * Gets a snapshot of this jobs state and performance. * @@ -960,50 +960,49 @@ final class DataSourceIngestJob { } /** - * Stores basic diagnostic statistics for an ingest job. + * Stores basic diagnostic statistics for a data source ingest job. */ class Snapshot { - private final long jobId; private final String dataSource; - private final long startTime; - private final DataSourceIngestPipeline.PipelineModule dataSourceLevelModule; - private final boolean fileIngestRunning; - private final Date fileIngestStartTime; + private final long jobId; + private final long jobStartTime; + private final long snapShotTime; + private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule; + private boolean fileIngestRunning; + private Date fileIngestStartTime; private final long processedFiles; private final long estimatedFilesToProcess; - private final long snapShotTime; private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; /** - * Constructs an object to store basic diagnostic statistics for an - * ingest job. + * Constructs an object to store basic diagnostic statistics for a data + * source ingest job. */ Snapshot() { - this.jobId = DataSourceIngestJob.this.id; this.dataSource = DataSourceIngestJob.this.dataSource.getName(); - this.startTime = DataSourceIngestJob.this.createTime; + this.jobId = DataSourceIngestJob.this.id; + this.jobStartTime = DataSourceIngestJob.this.createTime; + this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule(); /** - * Get a snapshot of data source level ingest. + * Determine whether file ingest is running at the time of this + * snapshot and determine the earliest file ingest level pipeline + * start time, if file ingest was started at all. */ - this.dataSourceLevelModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule(); - - /** - * Get a snapshot of file level ingest. - */ - boolean fileIngestIsRunning = false; - Date fileIngestStart = null; for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) { if (pipeline.isRunning()) { - fileIngestIsRunning = true; - if (null == fileIngestStart || pipeline.getStartTime().before(fileIngestStart)) { - fileIngestStart = pipeline.getStartTime(); - } + this.fileIngestRunning = true; + } + Date pipelineStartTime = pipeline.getStartTime(); + if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(this.fileIngestStartTime))) { + this.fileIngestStartTime = pipelineStartTime; } } - this.fileIngestRunning = fileIngestIsRunning; - this.fileIngestStartTime = fileIngestStart; + + /** + * Get processed file statistics. + */ synchronized (DataSourceIngestJob.this.fileIngestProgressLock) { this.processedFiles = DataSourceIngestJob.this.processedFiles; this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess; @@ -1017,13 +1016,13 @@ final class DataSourceIngestJob { } /** - * Gets the identifier of the ingest job that is the subject of this - * snapshot. + * Gets time these statistics were collected. * - * @return The ingest job id. + * @return The statistics collection time as number of milliseconds + * since January 1, 1970, 00:00:00 GMT. */ - long getJobId() { - return this.jobId; + long getSnapshotTime() { + return snapShotTime; } /** @@ -1037,13 +1036,13 @@ final class DataSourceIngestJob { } /** - * Gets files per second throughput since the ingest job that is the - * subject of this snapshot started. + * Gets the identifier of the ingest job that is the subject of this + * snapshot. * - * @return Files processed per second (approximate). + * @return The ingest job id. */ - double getSpeed() { - return (double) processedFiles / ((snapShotTime - startTime) / 1000); + long getJobId() { + return this.jobId; } /** @@ -1052,18 +1051,30 @@ final class DataSourceIngestJob { * @return The start time as number of milliseconds since January 1, * 1970, 00:00:00 GMT. */ - long getStartTime() { - return startTime; + long getJobStartTime() { + return jobStartTime; + } + + DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() { + return this.dataSourceLevelIngestModule; + } + + boolean fileIngestIsRunning() { + return this.fileIngestRunning; + } + + Date fileIngestStartTime() { + return this.fileIngestStartTime; } /** - * Gets time these statistics were collected. + * Gets files per second throughput since the ingest job that is the + * subject of this snapshot started. * - * @return The statistics collection time as number of milliseconds - * since January 1, 1970, 00:00:00 GMT. + * @return Files processed per second (approximate). */ - long getSnapshotTime() { - return snapShotTime; + double getSpeed() { + return (double) processedFiles / ((snapShotTime - jobStartTime) / 1000); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 6dcc3ce0bc..2cd1f65eb0 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -29,32 +29,27 @@ import org.sleuthkit.datamodel.Content; * source ingest job. It starts the modules, runs data sources through them, and * shuts them down when data source level ingest is complete. *

    - * This class is not thread-safe. + * This class is thread-safe. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); - private volatile PipelineModule currentModule; private boolean running; + private volatile PipelineModule currentModule; /** * Constructs an object that manages a sequence of data source level ingest * modules. It starts the modules, runs data sources through them, and shuts * them down when data source level ingest is complete. * - * @param job The data source ingest job to which this pipeline belongs. - * @param moduleTemplates The ingest module templates that define the - * pipeline. + * @param job The data source ingest job that owns this pipeline. + * @param moduleTemplates Templates for the creating the ingest modules that + * make up this pipeline. */ DataSourceIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - - /** - * Create a data source level ingest module instance from each ingest - * module template. - */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); @@ -73,11 +68,11 @@ final class DataSourceIngestPipeline { } /** - * Starts up the ingest module in this pipeline. + * Starts up the ingest modules in this pipeline. * * @return A list of ingest module startup errors, possibly empty. */ - List startUp() { + synchronized List startUp() { if (this.running) { throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS } @@ -100,7 +95,7 @@ final class DataSourceIngestPipeline { * be processed. * @return A list of processing errors, possible empty. */ - List process(DataSourceIngestTask task) { + synchronized List process(DataSourceIngestTask task) { if (!this.running) { throw new IllegalStateException("Attempt to process with pipeline that is not running"); //NON-NLS } @@ -109,14 +104,13 @@ final class DataSourceIngestPipeline { Content dataSource = task.getDataSource(); for (PipelineModule module : modules) { try { - module.setStartTime(); this.currentModule = module; String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", module.getDisplayName(), dataSource.getName()); this.job.updateDataSourceIngestProgressBarDisplayName(displayName); this.job.switchDataSourceIngestProgressBarToIndeterminate(); - ingestManager.setIngestTaskProgress(task, module.getDisplayName()); + DataSourceIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(dataSource, new DataSourceIngestModuleProgress(this.job)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); @@ -135,7 +129,7 @@ final class DataSourceIngestPipeline { /** * Gets the currently running module. * - * @return The module, possibly null. + * @return The module, possibly null if no module is currently running. */ PipelineModule getCurrentlyRunningModule() { return this.currentModule; @@ -143,17 +137,17 @@ final class DataSourceIngestPipeline { /** * This class decorates a data source level ingest module with a display - * name and a start time. + * name and a processing start time. */ static class PipelineModule implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; - private Date startTime; + private volatile Date processingStartTime; /** * Constructs an object that decorates a data source level ingest module - * with a display name and a running time. + * with a display name and a processing start time. * * @param module The data source level ingest module to be decorated. * @param displayName The display name. @@ -161,7 +155,7 @@ final class DataSourceIngestPipeline { PipelineModule(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; - this.startTime = new Date(); + this.processingStartTime = new Date(); } /** @@ -174,7 +168,7 @@ final class DataSourceIngestPipeline { } /** - * Gets a module name suitable for display in a UI. + * Gets the display of the decorated ingest module. * * @return The display name. */ @@ -182,21 +176,15 @@ final class DataSourceIngestPipeline { return this.displayName; } - /** - * Sets the start time to the current time. - */ - void setStartTime() { - this.startTime = new Date(); - } - /** * Gets the time the decorated ingest module started processing the data * source. * - * @return The start time. + * @return The start time, will be null if the module has not started + * processing the data source yet. */ - Date getStartTime() { - return this.startTime; + Date getProcessingStartTime() { + return this.processingStartTime; } /** @@ -212,7 +200,7 @@ final class DataSourceIngestPipeline { */ @Override public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - this.startTime = new Date(); + this.processingStartTime = new Date(); return this.module.process(dataSource, statusHelper); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index f322913690..8a0b2325f4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -28,9 +28,7 @@ import org.sleuthkit.datamodel.AbstractFile; * ingest job. It starts the modules, runs files through them, and shuts them * down when file level ingest is complete. *

    - * This class is not thread-safe, it is intended to be used by one file ingest - * thread at at time. However, the running flag is volatile since it may be read - * by another thread looking for a progress snapshot. + * This class is thread-safe. */ final class FileIngestPipeline { @@ -51,11 +49,6 @@ final class FileIngestPipeline { */ FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { this.job = job; - - /** - * Create an ingest module instance from each file ingest module - * template and add it to the pipeline. - */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); @@ -65,7 +58,7 @@ final class FileIngestPipeline { } /** - * Indicates whether or not there are any ingest modules in this pipeline. + * Queries whether or not there are any ingest modules in this pipeline. * * @return True or false. */ @@ -74,7 +67,7 @@ final class FileIngestPipeline { } /** - * Queries whether or not this file ingest level pipeline is running. + * Queries whether or not this pipeline is running. * * @return True or false. */ @@ -82,18 +75,28 @@ final class FileIngestPipeline { return this.running; } + /** + * Returns the start up time of this pipeline. + * + * @return The file processing start time, may be null if this pipeline has + * not been started yet. + */ + Date getStartTime() { + return this.startTime; + } + /** * Starts up all of the ingest modules in the pipeline. * * @return List of start up errors, possibly empty. */ - List startUp() { + synchronized List startUp() { if (this.running) { throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS } this.running = true; this.startTime = new Date(); - + List errors = new ArrayList<>(); for (PipelineModule module : this.modules) { try { @@ -105,22 +108,13 @@ final class FileIngestPipeline { return errors; } - /** - * Returns the start up time of the pipeline. - * - * @return The file processing start time, may be null. - */ - Date getStartTime() { - return this.startTime; - } - /** * Runs a file through the ingest modules in sequential order. * * @param task A file level ingest task containing a file to be processed. * @return A list of processing errors, possible empty. */ - List process(FileIngestTask task) { + synchronized List process(FileIngestTask task) { if (!this.running) { throw new IllegalStateException("Attempt to process file with pipeline that is not running"); //NON-NLS } @@ -151,7 +145,7 @@ final class FileIngestPipeline { * * @return A list of shut down errors, possibly empty. */ - List shutDown() { + synchronized List shutDown() { if (!this.running) { throw new IllegalStateException("Attempt to shut down a pipeline that is not running"); //NON-NLS } @@ -198,7 +192,7 @@ final class FileIngestPipeline { } /** - * Gets display name of the decorated ingest module. + * Gets the display name of the decorated ingest module. * * @return The display name. */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 01ebfb0765..d6725b8220 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -30,6 +30,8 @@ import org.sleuthkit.datamodel.Content; /** * Runs a collection of data sources through a set of ingest modules specified * via ingest job settings. + *

    + * This class is thread-safe. */ public final class IngestJob { @@ -44,13 +46,14 @@ public final class IngestJob { * * @param dataSources The data sources to be ingested. * @param settings The ingest job settings. - * @param doUI Whether or not to do UI interactions. + * @param runInteractively Whether or not this job should use progress bars, + * message boxes for errors, etc. */ - IngestJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { + IngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { this.id = IngestJob.nextId.getAndIncrement(); this.dataSourceJobs = new HashMap<>(); for (Content dataSource : dataSources) { - DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, doUI); + DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, runInteractively); this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); } } @@ -65,40 +68,73 @@ public final class IngestJob { } /** - * Gets a snapshot of the state and performance of this ingest job. + * Checks to see if this ingest job has at least one ingest pipeline when + * its settings are applied. + * + * @return True or false. + */ + boolean hasIngestPipeline() { + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + if (dataSourceJob.hasIngestPipeline()) { + return true; + } + } + return false; + } + + /** + * Starts this ingest job by starting its ingest module pipelines and + * scheduling the ingest tasks that make up the job. + * + * @return A collection of ingest module start up errors, empty on success. + */ + synchronized List start() { + List errors = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + errors.addAll(dataSourceJob.start()); + if (!errors.isEmpty()) { + // RJCTODO: Need to let sucessfully started data source ingest + // jobs know they should shut down. + break; + } + } + return errors; + } + + /** + * Gets a snapshot of the progress of this ingest job. * * @return The snapshot. */ synchronized public ProgressSnapshot getSnapshot() { - /** - * There are race conditions in the code that follows, but they are not - * important because this is just a coarse-grained status report. If - * stale data is returned in any single snapshot, it will be corrected - * in subsequent snapshots. - */ DataSourceIngestModuleHandle moduleHandle = null; - boolean fileIngestRunning = false; + boolean fileIngestIsRunning = false; Date fileIngestStartTime = null; - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - DataSourceIngestPipeline.PipelineModule module = dataSourceJob.getCurrentDataSourceIngestModule(); - if (null != module) { - moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); + for (DataSourceIngestJob.Snapshot snapshot : this.getDataSourceIngestJobSnapshots()) { + if (null != moduleHandle) { + DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); + if (null != module) { + moduleHandle = new DataSourceIngestModuleHandle(this.dataSourceJobs.get(snapshot.getJobId()), module); + } + } + if (snapshot.fileIngestIsRunning()) { + fileIngestIsRunning = true; + } + Date childFileIngestStartTime = snapshot.fileIngestStartTime(); + if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) { + fileIngestStartTime = childFileIngestStartTime; } - - // RJCTODO: For each data source job, check for a running flag and - // get the oldest start data for the start dates, if any. } - - return new ProgressSnapshot(moduleHandle, fileIngestRunning, fileIngestStartTime, this.cancelled); + return new ProgressSnapshot(moduleHandle, fileIngestIsRunning, fileIngestStartTime, this.cancelled); } /** - * Gets snapshots of the state and performance of this ingest job's child - * data source ingest jobs. + * Gets snapshots of the progress of each of this ingest job's child data + * source ingest jobs. * * @return A list of data source ingest job progress snapshots. */ - synchronized List getDetailedSnapshot() { // RJCTODO: Consider renaming + synchronized List getDataSourceIngestJobSnapshots() { List snapshots = new ArrayList<>(); for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { snapshots.add(dataSourceJob.getSnapshot()); @@ -107,21 +143,10 @@ public final class IngestJob { } /** - * Requests cancellation of a specific data source level ingest module. - * Returns immediately, but there may be a delay before the ingest module - * responds by stopping processing, if it is still running when the request - * is made. - * - * @param module The handle of the data source ingest module to be canceled, - * which can obtained from a progress snapshot. - */ - -// RJCTODO - - /** - * Requests cancellation of the data source level and file level ingest - * modules of this ingest job. Returns immediately, but there may be a delay - * before all of the ingest modules respond by stopping processing. + * Requests cancellation of this ingest job, which means discarding + * unfinished tasks and stopping the ingest pipelines. Returns immediately, + * but there may be a delay before all of the ingest modules in the + * pipelines respond by stopping processing. */ synchronized public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { @@ -131,8 +156,8 @@ public final class IngestJob { } /** - * Queries whether or not cancellation of the data source level and file - * level ingest modules of this ingest job has been requested. + * Queries whether or not cancellation of this ingest job has been + * requested. * * @return True or false. */ @@ -141,7 +166,20 @@ public final class IngestJob { } /** - * A snapshot of ingest job progress. + * Provides a callback for completed data source ingest jobs, allowing this + * ingest job to notify the ingest manager when it is complete. + * + * @param dataSourceIngestJob A completed data source ingest job. + */ + synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { + this.dataSourceJobs.remove(dataSourceIngestJob.getId()); + if (this.dataSourceJobs.isEmpty()) { + IngestManager.getInstance().finishIngestJob(this); + } + } + + /** + * A snapshot of the progress of an ingest job. */ public static final class ProgressSnapshot { @@ -154,11 +192,11 @@ public final class IngestJob { * Constructs a snapshot of ingest job progress. * * @param dataSourceModule The currently running data source level - * ingest module, may be null + * ingest module, may be null. * @param fileIngestRunning Whether or not file ingest is currently * running. * @param fileIngestStartTime The start time of file level ingest, may - * be null + * be null. * @param cancelled Whether or not a cancellation request has been * issued. */ @@ -171,21 +209,17 @@ public final class IngestJob { /** * Gets a handle to the currently running data source level ingest - * module at the time the snapshot is taken. + * module at the time the snapshot was taken. * * @return The handle, may be null. */ public DataSourceIngestModuleHandle runningDataSourceIngestModule() { - /** - * It is safe to hand out this reference because the object is - * immutable. - */ return this.dataSourceModule; } /** - * Queries whether or not file level ingest is running at the time the - * snapshot is taken. + * Queries whether or not file level ingest was running at the time the + * snapshot was taken. * * @return True or false. */ @@ -203,7 +237,8 @@ public final class IngestJob { } /** - * Queries whether or not a cancellation request has been issued. + * Queries whether or not a cancellation request had been issued at the + * time the snapshot was taken. * * @return True or false. */ @@ -215,21 +250,25 @@ public final class IngestJob { /** * A handle to a data source level ingest module that can be used to get - * basic information about the module and to request cancellation, i.e., - * shut down, of the module. + * basic information about the module and to request cancellation of the + * module. */ public static class DataSourceIngestModuleHandle { - private final long dataSourceIngestJobId; + private final DataSourceIngestJob job; private final DataSourceIngestPipeline.PipelineModule module; /** * Constructs a handle to a data source level ingest module that can be * used to get basic information about the module and to request * cancellation of the module. + * + * @param DataSourceIngestJob The data source ingest job that owns the + * data source level ingest module. + * @param module The data source level ingest module. */ - private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.PipelineModule module) { - this.dataSourceIngestJobId = dataSourceIngestJobId; + private DataSourceIngestModuleHandle(DataSourceIngestJob job, DataSourceIngestPipeline.PipelineModule module) { + this.job = job; this.module = module; } @@ -247,60 +286,34 @@ public final class IngestJob { * Returns the time the data source level ingest module associated with * this handle began processing. * - * @return The module start time. + * @return The module processing start time. */ public Date startTime() { - return this.module.getStartTime(); + return this.module.getProcessingStartTime(); } - + + /** + * Requests cancellation of the ingest module associated with this + * handle. Returns immediately, but there may be a delay before the + * ingest module responds by stopping processing. + */ public void cancel() { - // RJCTODO: - } - - } - - /** - * Starts up the ingest pipelines and ingest progress bars for this job. - * - * @return A collection of ingest module start up errors, empty on success. - */ - List start() { - boolean hasIngestPipeline = false; - List errors = new ArrayList<>(); - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - errors.addAll(dataSourceJob.start()); - hasIngestPipeline = dataSourceJob.hasIngestPipeline(); - } - return errors; - } - - /** - * Checks to see if this ingest job has at least one ingest pipeline. - * - * @return True or false. - */ - boolean hasIngestPipeline() { - boolean hasIngestPipeline = false; - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - if (dataSourceJob.hasIngestPipeline()) { - hasIngestPipeline = true; - break; + /** + * TODO: Cancellation needs to be more precise. The long-term + * solution is to add a cancel() method to IngestModule and do away + * with the cancellation queries of IngestJobContext. However, until + * an API change is legal, a cancel() method can be added to the + * DataSourceIngestModuleAdapter and FileIngestModuleAdapter classes + * and an instanceof check can be used to call it, with this code as + * the default implementation and the fallback. All of the ingest + * modules participating in this workaround will need to consult the + * cancelled flag in the adapters. + */ + if (this.job.getCurrentDataSourceIngestModule() == this.module) { + this.job.cancelCurrentDataSourceIngestModule(); } } - return hasIngestPipeline; - } - /** - * Provides a callback for completed data source ingest jobs, allowing the - * ingest job to notify the ingest manager when it is complete. - * - * @param dataSourceIngestJob A completed data source ingest job. - */ - synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { - this.dataSourceJobs.remove(dataSourceIngestJob.getId()); - if (this.dataSourceJobs.isEmpty()) { - IngestManager.getInstance().finishJob(this); - } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java index 63ed560b30..258e01c5ad 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java @@ -87,6 +87,6 @@ public final class IngestJobConfigurator { */ @Deprecated public void startIngestJobs(List dataSources) { - IngestManager.getInstance().startIngestJob(dataSources, this.settings, true); + IngestManager.getInstance().queueIngestJob(dataSources, this.settings, true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index d1c56e3967..53be42ce12 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -56,15 +56,17 @@ public class IngestManager { private static IngestManager instance = null; /** - * The ingest manager maintains a mapping of ingest job IDs to ingest jobs. + * The ingest manager maintains a mapping of ingest job IDs to running + * ingest jobs. This, in combination with a mapping of thread IDs to + * Callable ingest job starters, determines whether or not ingest is + * running. */ - private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); + private final HashMap jobsById = new HashMap<>(); /** * Each runnable/callable task the ingest manager submits to its thread * pools is given a unique thread/task ID. */ - // TODO: It is no longer necessary to have multiple thread pools. private final AtomicLong nextThreadId = new AtomicLong(0L); /** @@ -73,7 +75,7 @@ public class IngestManager { * ingest job starter is maintained to provide handles that can be used to * cancel the ingest job starter. */ - private final ConcurrentHashMap> ingestJobStarters = new ConcurrentHashMap<>(); + private final HashMap> ingestJobStarters = new HashMap<>(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); /** @@ -83,11 +85,11 @@ public class IngestManager { * ingest task executers. There is a single data source level ingest thread * and a user configurable number of file level ingest threads. */ + private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; - private int numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; - private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); + private int numberOfFileIngestThreads; private final ExecutorService fileIngestThreadPool; /** @@ -202,6 +204,31 @@ public class IngestManager { return instance; } + /** + * Constructs a manager of the creation and execution of ingest jobs, i.e., + * the processing of data sources by ingest modules. The manager immediately + * submits ingest task executers (Callable objects) to the data source level + * ingest and file level ingest thread pools. The ingest task executers are + * simple consumers that will normally run as long as the application runs. + */ + private IngestManager() { + long threadId = nextThreadId.incrementAndGet(); + dataSourceIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); + ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + + numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); + if ((numberOfFileIngestThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfFileIngestThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { + numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; + UserPreferences.setNumberOfFileIngestThreads(numberOfFileIngestThreads); + } + fileIngestThreadPool = Executors.newFixedThreadPool(numberOfFileIngestThreads); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + threadId = nextThreadId.incrementAndGet(); + fileIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); + ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + } + } + /** * Gets the number of file ingest threads the ingest manager will use to do * ingest jobs. @@ -212,17 +239,127 @@ public class IngestManager { return numberOfFileIngestThreads; } + private void subscribeToCaseEvents() { + Case.addPropertyChangeListener(new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent event) { + if (event.getPropertyName().equals(Case.Events.CURRENT_CASE.toString())) { + if (event.getNewValue() != null) { + handleCaseOpened(); + } else { + handleCaseClosed(); + } + } + } + }); + } + + void handleCaseOpened() { + this.jobCreationIsEnabled = true; + clearIngestMessageBox(); + } + + void handleCaseClosed() { + this.jobCreationIsEnabled = false; + cancelAllIngestJobs(); + clearIngestMessageBox(); + } + + /** + * Called by the custom installer for this package once the window system is + * initialized, allowing the ingest manager to get the top component used to + * display ingest messages. + */ + void initIngestMessageInbox() { + ingestMessageBox = IngestMessageTopComponent.findInstance(); + } + + /** + * Post a message to the ingest messages in box. + * + * @param message The message to be posted. + */ + // RJCTODO: Can I cut this off effectively? + void postIngestMessage(IngestMessage message) { + if (ingestMessageBox != null) { + if (message.getMessageType() != IngestMessage.MessageType.ERROR && message.getMessageType() != IngestMessage.MessageType.WARNING) { + ingestMessageBox.displayMessage(message); + } else { + long errorPosts = ingestErrorMessagePosts.incrementAndGet(); + if (errorPosts <= MAX_ERROR_MESSAGE_POSTS) { + ingestMessageBox.displayMessage(message); + } else if (errorPosts == MAX_ERROR_MESSAGE_POSTS + 1) { + IngestMessage errorMessageLimitReachedMessage = IngestMessage.createErrorMessage( + NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.title"), + NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.subject"), + NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.msg", MAX_ERROR_MESSAGE_POSTS)); + ingestMessageBox.displayMessage(errorMessageLimitReachedMessage); + } + } + } + } + + private void clearIngestMessageBox() { + if (ingestMessageBox != null) { + ingestMessageBox.clearMessages(); + } + ingestErrorMessagePosts.set(0); + } + + /** + * Queues an ingest job that will process a collection of data sources. The + * job will be started on a worker thread. + * + * @param dataSources The data sources to process. + * @param settings The settings for the ingest job. + * @param runInteractively Whether or not this job should use progress bars, + * message boxes for errors, etc. + */ + public synchronized void queueIngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { + if (this.jobCreationIsEnabled) { + IngestJob job = new IngestJob(dataSources, settings, runInteractively); + if (job.hasIngestPipeline()) { + long taskId = nextThreadId.incrementAndGet(); + Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, job, runInteractively)); + ingestJobStarters.put(taskId, task); + } + } + } + + /** + * Starts an ingest job that will process a collection of data sources. + * + * @param dataSources The data sources to process. + * @param settings The settings for the ingest job. + * @param runInteractively Whether or not this job should use progress bars, + * message boxes for errors, etc. + * @return The ingest job that was started or null if the job could not be + * started. + */ + public synchronized IngestJob startIngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { + IngestJob job = null; + if (this.jobCreationIsEnabled) { + job = new IngestJob(dataSources, settings, runInteractively); + if (job.hasIngestPipeline()) { + List errors = this.startIngestJob(job, runInteractively); + if (!errors.isEmpty()) { + job = null; + } + } + } + return job; + } + /** * Starts an ingest job for a collection of data sources. * - * @param dataSources The data sources to be processed. - * @param settings The ingest job settings. - * @param doUI Whether or not to support user interaction, e.g., showing - * message boxes and reporting progress through the NetBeans Progress API. - * @return The ingest job that was started + * @param dataSource The data sources to ingest. + * @param settings The settings for the job. + * @param runInteractively Whether or not to interact with the UI + * @return A collection of ingest module start up errors, empty on success. */ - public synchronized void startIngestJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { - if (!isIngestRunning()) { + private List startIngestJob(IngestJob job, boolean runInteractively) { + if (runInteractively && jobsById.isEmpty()) { // RJCTODO: This is sort of broken clearIngestMessageBox(); } @@ -230,16 +367,25 @@ public class IngestManager { ingestMonitor.start(); } - if (doUI) { - /** - * Assume request is from code running on the EDT and dispatch to a - * worker thread. - */ - long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, dataSources, settings, doUI)); - ingestJobStarters.put(taskId, task); + List errors = job.start(); + if (errors.isEmpty()) { + long jobId = job.getId(); + this.jobsById.put(jobId, job); + this.fireIngestJobStarted(jobId); + IngestManager.logger.log(Level.INFO, "Ingest job {0} started", jobId); + } + return errors; + } + + void finishIngestJob(IngestJob job) { + long jobId = job.getId(); + this.jobsById.remove(jobId); + if (!job.isCancelled()) { + IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); + this.fireIngestJobCompleted(jobId); } else { - this.startJob(dataSources, settings, doUI); + IngestManager.logger.log(Level.INFO, "Ingest job {0} cancelled", jobId); + this.fireIngestJobCancelled(jobId); } } @@ -249,9 +395,7 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { - // RJCTODO: This may return the wrong answer if an IngestJobStarter has - // been dispatched to the startIngestJobsThreadPool. - return !this.jobsById.isEmpty(); + return !this.jobsById.isEmpty() || !ingestJobStarters.values().isEmpty(); } /** @@ -332,123 +476,59 @@ public class IngestManager { } /** - * Constructs a manager of the creation and execution of ingest jobs, i.e., - * the processing of data sources by ingest modules. The manager immediately - * submits ingest task executers (Callable objects) to the data source level - * ingest and file level ingest thread pools. The ingest task executers are - * simple consumers that will normally run as long as the application runs. - */ - private IngestManager() { - startDataSourceIngestThread(); - - numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); - if ((numberOfFileIngestThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfFileIngestThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { - numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; - UserPreferences.setNumberOfFileIngestThreads(numberOfFileIngestThreads); - } - fileIngestThreadPool = Executors.newFixedThreadPool(numberOfFileIngestThreads); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - startFileIngestThread(); - } - } - - /** - * Called by the custom installer for this package once the window system is - * initialized, allowing the ingest manager to get the top component used to - * display ingest messages. - */ - void initIngestMessageInbox() { - ingestMessageBox = IngestMessageTopComponent.findInstance(); - } - - /** - * Submits an ingest task executer Callable to the data source level ingest - * thread pool. - */ - private void startDataSourceIngestThread() { - long threadId = nextThreadId.incrementAndGet(); - dataSourceIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); - ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); - } - - /** - * Submits a ingest task executer Callable to the file level ingest thread - * pool. - */ - private void startFileIngestThread() { - long threadId = nextThreadId.incrementAndGet(); - fileIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); - ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); - } - - private void subscribeToCaseEvents() { - Case.addPropertyChangeListener(new PropertyChangeListener() { - @Override - public void propertyChange(PropertyChangeEvent event) { - if (event.getPropertyName().equals(Case.Events.CURRENT_CASE.toString())) { - if (event.getNewValue() != null) { - handleCaseOpened(); - } else { - handleCaseClosed(); - } - } - } - }); - } - - void handleCaseOpened() { - this.jobCreationIsEnabled = true; - clearIngestMessageBox(); - } - - void handleCaseClosed() { - this.jobCreationIsEnabled = false; - cancelAllIngestJobs(); - clearIngestMessageBox(); - } - - private void clearIngestMessageBox() { - if (ingestMessageBox != null) { - ingestMessageBox.clearMessages(); - } - ingestErrorMessagePosts.set(0); - } - - /** - * Starts an ingest job for a collection of data sources. + * Fire an ingest event signifying an ingest job started. * - * @param dataSource The data sources to ingest. - * @param settings The settings for the job. - * @param doUI Whether or not to interact with the UI - * @return A collection of ingest module start up errors, empty on success. + * @param ingestJobId The ingest job id. */ - private List startJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { - List errors = new ArrayList<>(); - if (this.jobCreationIsEnabled) { - IngestJob job = new IngestJob(dataSources, settings, doUI); - long jobId = job.getId(); - this.jobsById.put(jobId, job); - errors = job.start(); - if (errors.isEmpty() && job.hasIngestPipeline()) { - this.fireIngestJobStarted(jobId); - IngestManager.logger.log(Level.INFO, "Ingest job {0} started", jobId); - } else { - this.jobsById.remove(jobId); - } - } - return errors; + void fireIngestJobStarted(long ingestJobId) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); } - void finishJob(IngestJob job) { - long jobId = job.getId(); - this.jobsById.remove(jobId); - if (!job.isCancelled()) { - IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); - this.fireIngestJobCompleted(jobId); - } else { - IngestManager.logger.log(Level.INFO, "Ingest job {0} cancelled", jobId); - this.fireIngestJobCancelled(jobId); - } + /** + * Fire an ingest event signifying an ingest job finished. + * + * @param ingestJobId The ingest job id. + */ + void fireIngestJobCompleted(long ingestJobId) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); + } + + /** + * Fire an ingest event signifying an ingest job was canceled. + * + * @param ingestJobId The ingest job id. + */ + void fireIngestJobCancelled(long ingestJobId) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); + } + + /** + * Fire an ingest event signifying the ingest of a file is completed. + * + * @param file The file that is completed. + */ + void fireFileIngestDone(AbstractFile file) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); + } + + /** + * Fire an event signifying a blackboard post by an ingest module. + * + * @param moduleDataEvent A ModuleDataEvent with the details of the posting. + */ + void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); + } + + /** + * Fire an event signifying discovery of additional content by an ingest + * module. + * + * @param moduleDataEvent A ModuleContentEvent with the details of the new + * content. + */ + void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); } /** @@ -540,83 +620,16 @@ public class IngestManager { } /** - * Fire an ingest event signifying an ingest job started. + * Gets snapshots of the state of all running ingest jobs. * - * @param ingestJobId The ingest job id. + * @return A list of ingest job state snapshots. */ - void fireIngestJobStarted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); - } - - /** - * Fire an ingest event signifying an ingest job finished. - * - * @param ingestJobId The ingest job id. - */ - void fireIngestJobCompleted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); - } - - /** - * Fire an ingest event signifying an ingest job was canceled. - * - * @param ingestJobId The ingest job id. - */ - void fireIngestJobCancelled(long ingestJobId) { - fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); - } - - /** - * Fire an ingest event signifying the ingest of a file is completed. - * - * @param file The file that is completed. - */ - void fireFileIngestDone(AbstractFile file) { - fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); - } - - /** - * Fire an event signifying a blackboard post by an ingest module. - * - * @param moduleDataEvent A ModuleDataEvent with the details of the posting. - */ - void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { - fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); - } - - /** - * Fire an event signifying discovery of additional content by an ingest - * module. - * - * @param moduleDataEvent A ModuleContentEvent with the details of the new - * content. - */ - void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { - fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); - } - - /** - * Post a message to the ingest messages in box. - * - * @param message The message to be posted. - */ - void postIngestMessage(IngestMessage message) { - if (ingestMessageBox != null) { - if (message.getMessageType() != IngestMessage.MessageType.ERROR && message.getMessageType() != IngestMessage.MessageType.WARNING) { - ingestMessageBox.displayMessage(message); - } else { - long errorPosts = ingestErrorMessagePosts.incrementAndGet(); - if (errorPosts <= MAX_ERROR_MESSAGE_POSTS) { - ingestMessageBox.displayMessage(message); - } else if (errorPosts == MAX_ERROR_MESSAGE_POSTS + 1) { - IngestMessage errorMessageLimitReachedMessage = IngestMessage.createErrorMessage( - NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.title"), - NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.subject"), - NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.msg", MAX_ERROR_MESSAGE_POSTS)); - ingestMessageBox.displayMessage(errorMessageLimitReachedMessage); - } - } + List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); + for (IngestJob job : this.jobsById.values()) { + snapShots.addAll(job.getDataSourceIngestJobSnapshots()); } + return snapShots; } /** @@ -633,73 +646,56 @@ public class IngestManager { } } - /** - * Gets snapshots of the state of all running ingest jobs. - * - * @return A list of ingest job state snapshots. - */ - List getIngestJobSnapshots() { - List snapShots = new ArrayList<>(); - for (IngestJob job : this.jobsById.values()) { - snapShots.addAll(job.getDetailedSnapshot()); - } - return snapShots; - } - /** * Creates and starts an ingest job for a collection of data sources. */ private final class IngestJobStarter implements Callable { private final long threadId; - private final Collection dataSources; - private final IngestJobSettings settings; - private final boolean doStartupErrorsMsgBox; + private final IngestJob job; + private final boolean runInteractively; private ProgressHandle progress; - IngestJobStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { + IngestJobStarter(long threadId, IngestJob job, boolean runInteractively) { this.threadId = threadId; - this.dataSources = dataSources; - this.settings = settings; - this.doStartupErrorsMsgBox = doMessageDialogs; + this.job = job; + this.runInteractively = runInteractively; } @Override public Void call() { try { - /** - * Bail out if there is nothing to do or cancellation has been - * requested. - */ - if (this.dataSources.isEmpty() || Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { return null; } /** * Set up a progress bar. */ - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.displayName"); - progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - if (progress != null) { - progress.setDisplayName(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.cancelling", - displayName)); + if (runInteractively) { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.displayName"); + this.progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (progress != null) { + progress.setDisplayName(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.cancelling", + displayName)); + } + Future handle = ingestJobStarters.remove(threadId); + handle.cancel(true); + return true; } - Future handle = ingestJobStarters.remove(threadId); - handle.cancel(true); - return true; - } - }); - progress.start(); + }); + progress.start(); + } /** * Try to start the ingest job. */ - List errors = IngestManager.this.startJob(this.dataSources, this.settings, true); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { + List errors = IngestManager.this.startIngestJob(job, runInteractively); + if (!errors.isEmpty() && this.runInteractively) { StringBuilder moduleStartUpErrors = new StringBuilder(); for (IngestModuleError error : errors) { String moduleName = error.getModuleDisplayName(); @@ -867,6 +863,7 @@ public class IngestManager { String getFileName() { return fileName; } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index 17bd6a0dbf..7485b71e4b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -81,4 +81,10 @@ public interface IngestModule { * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ void startUp(IngestJobContext context) throws IngestModuleException; + + /** + * TODO: The next time an API change is legal, add a cancel() method and + * remove the "ingest job is canceled" queries from the IngestJobContext + * class. + */ } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 0e9e97a96b..ce6f0e3c0a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -202,7 +202,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { break; case 2: SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); - cellValue = dateFormat.format(new Date(snapShot.getStartTime())); + cellValue = dateFormat.format(new Date(snapShot.getJobStartTime())); break; case 3: cellValue = snapShot.getFilesProcessed(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index b69e967ad9..9ed1f33489 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -199,7 +199,7 @@ public final class RunIngestModulesDialog extends JDialog { ingestJobSettings.save(); showWarnings(ingestJobSettings); if (startIngestJob) { - IngestManager.getInstance().startIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); + IngestManager.getInstance().queueIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); } setVisible(false); dispose(); From 1f42d3df07a9a2c91cdb06593555245211262f2a Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Wed, 17 Dec 2014 11:41:29 -0500 Subject: [PATCH 58/84] Fixed issue where string was displayed as all "SQUARES". --- .../modules/interestingitems/InterestingItemDefsPanel.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java index 3eddc9236d..a33077206c 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java @@ -18,7 +18,7 @@ */ package org.sleuthkit.autopsy.modules.interestingitems; -import java.awt.EventQueue; +import java.awt.*; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -532,7 +532,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp jTextArea1.setBackground(new java.awt.Color(240, 240, 240)); jTextArea1.setColumns(20); - jTextArea1.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N + jTextArea1.setFont(jTextArea1.getFont().deriveFont(Font.PLAIN, 11)); // NON-NLS jTextArea1.setLineWrap(true); jTextArea1.setRows(3); jTextArea1.setText(org.openide.util.NbBundle.getMessage(InterestingItemDefsPanel.class, "InterestingItemDefsPanel.jTextArea1.text")); // NOI18N From a2365c7b32525615e15a13a7fcc40ece6ee8019c Mon Sep 17 00:00:00 2001 From: Nick Davis Date: Wed, 17 Dec 2014 11:49:08 -0500 Subject: [PATCH 59/84] Switched import statement to be specific instead of importing all of java.awt. --- .../modules/interestingitems/InterestingItemDefsPanel.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java index a33077206c..a52f87ffd0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java @@ -18,7 +18,8 @@ */ package org.sleuthkit.autopsy.modules.interestingitems; -import java.awt.*; +import java.awt.EventQueue; +import java.awt.Font; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; From 4c8b967773204ec8ee4188218f4f0ea68230646a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 17 Dec 2014 13:17:01 -0500 Subject: [PATCH 60/84] Fix bug in ingest pipelines code --- .../autopsy/ingest/DataSourceIngestJob.java | 6 ++++-- .../autopsy/ingest/DataSourceIngestPipeline.java | 9 --------- .../autopsy/ingest/FileIngestPipeline.java | 14 +------------- .../org/netbeans/core/startup/Bundle.properties | 2 +- .../core/windows/view/ui/Bundle.properties | 2 +- 5 files changed, 7 insertions(+), 26 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 309a083050..1e320b2648 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -400,7 +400,9 @@ final class DataSourceIngestJob { this.stage = DataSourceIngestJob.Stages.FIRST; if (this.hasFileIngestPipeline()) { - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + synchronized (this.fileIngestProgressLock) { + this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + } } if (this.runInteractively) { @@ -658,7 +660,7 @@ final class DataSourceIngestJob { FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); if (!pipeline.isEmpty()) { AbstractFile file = task.getFile(); - ++this.processedFiles; + ++this.processedFiles; // RJCTODO: This was previously inside the lock if (this.runInteractively) { /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 2cd1f65eb0..0838abe2bc 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -36,7 +36,6 @@ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; private final List modules = new ArrayList<>(); - private boolean running; private volatile PipelineModule currentModule; /** @@ -73,10 +72,6 @@ final class DataSourceIngestPipeline { * @return A list of ingest module startup errors, possibly empty. */ synchronized List startUp() { - if (this.running) { - throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS - } - List errors = new ArrayList<>(); for (PipelineModule module : modules) { try { @@ -96,10 +91,6 @@ final class DataSourceIngestPipeline { * @return A list of processing errors, possible empty. */ synchronized List process(DataSourceIngestTask task) { - if (!this.running) { - throw new IllegalStateException("Attempt to process with pipeline that is not running"); //NON-NLS - } - List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); for (PipelineModule module : modules) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 8a0b2325f4..78db0b482d 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -91,12 +91,8 @@ final class FileIngestPipeline { * @return List of start up errors, possibly empty. */ synchronized List startUp() { - if (this.running) { - throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS - } - this.running = true; this.startTime = new Date(); - + this.running = true; List errors = new ArrayList<>(); for (PipelineModule module : this.modules) { try { @@ -115,10 +111,6 @@ final class FileIngestPipeline { * @return A list of processing errors, possible empty. */ synchronized List process(FileIngestTask task) { - if (!this.running) { - throw new IllegalStateException("Attempt to process file with pipeline that is not running"); //NON-NLS - } - List errors = new ArrayList<>(); AbstractFile file = task.getFile(); for (PipelineModule module : this.modules) { @@ -146,10 +138,6 @@ final class FileIngestPipeline { * @return A list of shut down errors, possibly empty. */ synchronized List shutDown() { - if (!this.running) { - throw new IllegalStateException("Attempt to shut down a pipeline that is not running"); //NON-NLS - } - List errors = new ArrayList<>(); for (PipelineModule module : this.modules) { try { diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index 575a34a2f6..28aa07b92a 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Mon, 15 Dec 2014 16:14:04 -0500 +#Wed, 17 Dec 2014 13:11:49 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index 9ca23c2a58..3a5c88bbe9 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Mon, 15 Dec 2014 16:14:04 -0500 +#Wed, 17 Dec 2014 13:11:49 -0500 CTL_MainWindow_Title=Autopsy 3.1.1 CTL_MainWindow_Title_No_Project=Autopsy 3.1.1 From ad886ad73654f98e5c21af6eee47ff7c380b1b88 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 17 Dec 2014 13:21:58 -0500 Subject: [PATCH 61/84] Fix bug in ingest pipelines code --- .../autopsy/ingest/DataSourceIngestJob.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 1e320b2648..a1408ef6c2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -660,13 +660,13 @@ final class DataSourceIngestJob { FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); if (!pipeline.isEmpty()) { AbstractFile file = task.getFile(); - ++this.processedFiles; // RJCTODO: This was previously inside the lock - if (this.runInteractively) { - /** - * Update the file ingest progress bar. - */ - synchronized (this.fileIngestProgressLock) { + synchronized (this.fileIngestProgressLock) { + ++this.processedFiles; + if (this.runInteractively) { + /** + * Update the file ingest progress bar. + */ if (this.processedFiles <= this.estimatedFilesToProcess) { this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); } else { From 04c81a76629bb1bbb9c602867f8d621d40863cbb Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 17 Dec 2014 15:57:09 -0500 Subject: [PATCH 62/84] Correct implementations of new IngestManager APIs --- .../AddImageWizardIngestConfigPanel.java | 2 +- .../autopsy/ingest/IngestJobConfigurator.java | 2 +- .../autopsy/ingest/IngestManager.java | 285 ++++++++++-------- .../ingest/RunIngestModulesDialog.java | 2 +- 4 files changed, 168 insertions(+), 123 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 7ac97c1ca2..fe70d57eeb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,7 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel dataSources) { - IngestManager.getInstance().queueIngestJob(dataSources, this.settings, true); + IngestManager.getInstance().queueIngestJob(dataSources, this.settings); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 53be42ce12..b0cb987710 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest; +import java.awt.EventQueue; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; @@ -53,30 +54,28 @@ import org.sleuthkit.datamodel.Content; public class IngestManager { private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); - private static IngestManager instance = null; + private static IngestManager instance; /** * The ingest manager maintains a mapping of ingest job IDs to running - * ingest jobs. This, in combination with a mapping of thread IDs to - * Callable ingest job starters, determines whether or not ingest is - * running. + * ingest jobs. */ - private final HashMap jobsById = new HashMap<>(); + private final ConcurrentHashMap jobsById; /** * Each runnable/callable task the ingest manager submits to its thread * pools is given a unique thread/task ID. */ - private final AtomicLong nextThreadId = new AtomicLong(0L); + private final AtomicLong nextThreadId; /** - * Ingest jobs are started on a pool thread by ingest job starters. A - * mapping of thread/task IDs to the result objects associated with each - * ingest job starter is maintained to provide handles that can be used to - * cancel the ingest job starter. + * Ingest jobs may be queued to be started on a pool thread by ingest job + * starters. A mapping of thread/task IDs to the result objects associated + * with each ingest job starter is maintained to provide handles that can be + * used to cancel the ingest job starter. */ - private final HashMap> ingestJobStarters = new HashMap<>(); - private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); + private final ConcurrentHashMap> ingestJobStarters; + private final ExecutorService startIngestJobsThreadPool; /** * Ingest jobs use an ingest task scheduler to break themselves down into @@ -85,7 +84,7 @@ public class IngestManager { * ingest task executers. There is a single data source level ingest thread * and a user configurable number of file level ingest threads. */ - private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); + private final ExecutorService dataSourceIngestThreadPool; private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; @@ -98,16 +97,16 @@ public class IngestManager { * events and ingest module events. Property changes are fired by ingest * event publishers on a pool thread. */ - private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); - private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); - private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); + private final PropertyChangeSupport ingestJobEventPublisher; + private final PropertyChangeSupport ingestModuleEventPublisher; + private final ExecutorService fireIngestEventsThreadPool; /** * The ingest manager uses an ingest monitor to determine when system * resources are under pressure. If the monitor detects such a situation, it * calls back to the ingest manager to cancel all ingest jobs in progress. */ - private final IngestMonitor ingestMonitor = new IngestMonitor(); + private final IngestMonitor ingestMonitor; /** * The ingest manager provides access to a top component that is used by @@ -116,15 +115,15 @@ public class IngestManager { */ private static final int MAX_ERROR_MESSAGE_POSTS = 200; private volatile IngestMessageTopComponent ingestMessageBox; - private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); + private final AtomicLong ingestErrorMessagePosts; /** * The ingest manager supports reporting of ingest processing progress by * collecting snapshots of the activities of the ingest threads, ingest job * progress, and ingest module run times. */ - private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. - private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); + private final ConcurrentHashMap ingestThreadActivitySnapshots; + private final ConcurrentHashMap ingestModuleRunTimes; /** * The ingest job creation capability of the ingest manager can be turned on @@ -132,6 +131,13 @@ public class IngestManager { */ private volatile boolean jobCreationIsEnabled; + /** + * The ingest manager can be directed to forgo use of message boxes, the + * ingest message box, NetBeans progress handles, etc. Running interactively + * is the default. + */ + private volatile boolean runInteractively; + /** * Ingest job events. */ @@ -194,10 +200,11 @@ public class IngestManager { */ public synchronized static IngestManager getInstance() { if (instance == null) { - // Two stage construction to avoid allowing "this" reference to - // escape from the constructor via the property change listener. - // This is to ensure that a partially constructed ingest manager is - // not published to other threads. + /** + * Two stage construction to avoid allowing the "this" reference to + * be prematurely published from the constructor via the Case + * property change listener. + */ instance = new IngestManager(); instance.subscribeToCaseEvents(); } @@ -208,13 +215,26 @@ public class IngestManager { * Constructs a manager of the creation and execution of ingest jobs, i.e., * the processing of data sources by ingest modules. The manager immediately * submits ingest task executers (Callable objects) to the data source level - * ingest and file level ingest thread pools. The ingest task executers are - * simple consumers that will normally run as long as the application runs. + * ingest and file level ingest thread pools. These ingest task executers + * are simple consumers that will normally run as long as the application + * runs. */ private IngestManager() { - long threadId = nextThreadId.incrementAndGet(); - dataSourceIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); - ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + this.runInteractively = true; + this.ingestModuleRunTimes = new ConcurrentHashMap<>(); + this.ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); + this.ingestErrorMessagePosts = new AtomicLong(0L); + this.ingestMonitor = new IngestMonitor(); + this.ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); + this.fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); + this.ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); + this.dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); + this.startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); + this.nextThreadId = new AtomicLong(0L); + this.jobsById = new ConcurrentHashMap<>(); + this.ingestJobStarters = new ConcurrentHashMap<>(); + + this.startDataSourceIngestThread(); numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); if ((numberOfFileIngestThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfFileIngestThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { @@ -223,20 +243,28 @@ public class IngestManager { } fileIngestThreadPool = Executors.newFixedThreadPool(numberOfFileIngestThreads); for (int i = 0; i < numberOfFileIngestThreads; ++i) { - threadId = nextThreadId.incrementAndGet(); - fileIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); - ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + startFileIngestThread(); } } /** - * Gets the number of file ingest threads the ingest manager will use to do - * ingest jobs. - * - * @return The number of file ingest threads. + * Submits an ingest task executer Callable to the data source level ingest + * thread pool. */ - public int getNumberOfFileIngestThreads() { - return numberOfFileIngestThreads; + private void startDataSourceIngestThread() { + long threadId = nextThreadId.incrementAndGet(); + dataSourceIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); + ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + } + + /** + * Submits a ingest task executer Callable to the file level ingest thread + * pool. + */ + private void startFileIngestThread() { + long threadId = nextThreadId.incrementAndGet(); + fileIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); + ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } private void subscribeToCaseEvents() { @@ -254,17 +282,29 @@ public class IngestManager { }); } - void handleCaseOpened() { + synchronized void handleCaseOpened() { this.jobCreationIsEnabled = true; clearIngestMessageBox(); } - void handleCaseClosed() { + synchronized void handleCaseClosed() { this.jobCreationIsEnabled = false; cancelAllIngestJobs(); clearIngestMessageBox(); } + /** + * The ingest manager can be directed to forgo use of message boxes, the + * ingest message box, NetBeans progress handles, etc. Running interactively + * is the default. + * + * @param runInteractively whether or not to this ingest manager should run + * ingest interactively. + */ + public synchronized void setRunInteractively(boolean runInteractively) { + this.runInteractively = runInteractively; + } + /** * Called by the custom installer for this package once the window system is * initialized, allowing the ingest manager to get the top component used to @@ -279,9 +319,8 @@ public class IngestManager { * * @param message The message to be posted. */ - // RJCTODO: Can I cut this off effectively? - void postIngestMessage(IngestMessage message) { - if (ingestMessageBox != null) { + synchronized void postIngestMessage(IngestMessage message) { + if (ingestMessageBox != null && this.runInteractively) { if (message.getMessageType() != IngestMessage.MessageType.ERROR && message.getMessageType() != IngestMessage.MessageType.WARNING) { ingestMessageBox.displayMessage(message); } else { @@ -306,21 +345,30 @@ public class IngestManager { ingestErrorMessagePosts.set(0); } + /** + * Gets the number of file ingest threads the ingest manager will use to do + * ingest jobs. + * + * @return The number of file ingest threads. + */ + public int getNumberOfFileIngestThreads() { + return numberOfFileIngestThreads; + } + /** * Queues an ingest job that will process a collection of data sources. The * job will be started on a worker thread. * * @param dataSources The data sources to process. * @param settings The settings for the ingest job. - * @param runInteractively Whether or not this job should use progress bars, - * message boxes for errors, etc. */ - public synchronized void queueIngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { + public synchronized void queueIngestJob(Collection dataSources, IngestJobSettings settings) { if (this.jobCreationIsEnabled) { - IngestJob job = new IngestJob(dataSources, settings, runInteractively); + IngestJob job = new IngestJob(dataSources, settings, this.runInteractively); if (job.hasIngestPipeline()) { + this.jobsById.put(job.getId(), job); long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, job, runInteractively)); + Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, job)); ingestJobStarters.put(taskId, task); } } @@ -331,53 +379,85 @@ public class IngestManager { * * @param dataSources The data sources to process. * @param settings The settings for the ingest job. - * @param runInteractively Whether or not this job should use progress bars, - * message boxes for errors, etc. - * @return The ingest job that was started or null if the job could not be - * started. + * @return The ingest job that was started on success or null on failure. */ - public synchronized IngestJob startIngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { - IngestJob job = null; + public synchronized IngestJob startIngestJob(Collection dataSources, IngestJobSettings settings) { if (this.jobCreationIsEnabled) { - job = new IngestJob(dataSources, settings, runInteractively); + IngestJob job = new IngestJob(dataSources, settings, this.runInteractively); if (job.hasIngestPipeline()) { - List errors = this.startIngestJob(job, runInteractively); - if (!errors.isEmpty()) { - job = null; + this.jobsById.put(job.getId(), job); + if (this.startIngestJob(job)) { + return job; } } } - return job; + return null; } /** * Starts an ingest job for a collection of data sources. * - * @param dataSource The data sources to ingest. - * @param settings The settings for the job. - * @param runInteractively Whether or not to interact with the UI - * @return A collection of ingest module start up errors, empty on success. + * @param job The ingest job to start. + * @return True if the job was started, false otherwise. */ - private List startIngestJob(IngestJob job, boolean runInteractively) { - if (runInteractively && jobsById.isEmpty()) { // RJCTODO: This is sort of broken - clearIngestMessageBox(); - } + private boolean startIngestJob(IngestJob job) { + boolean success = false; - if (!ingestMonitor.isRunning()) { - ingestMonitor.start(); - } + if (this.jobCreationIsEnabled) { + if (runInteractively && jobsById.isEmpty()) { // RJCTODO: This is sort of broken + clearIngestMessageBox(); + } - List errors = job.start(); - if (errors.isEmpty()) { - long jobId = job.getId(); - this.jobsById.put(jobId, job); - this.fireIngestJobStarted(jobId); - IngestManager.logger.log(Level.INFO, "Ingest job {0} started", jobId); + if (!ingestMonitor.isRunning()) { + ingestMonitor.start(); + } + + List errors = job.start(); + if (errors.isEmpty()) { + this.fireIngestJobStarted(job.getId()); + IngestManager.logger.log(Level.INFO, "Ingest job {0} started", job.getId()); + success = true; + } else { + this.jobsById.remove(job.getId()); + IngestManager.logger.log(Level.INFO, "Ingest job {0} could not be started", job.getId()); + if (this.runInteractively) { + EventQueue.invokeLater(new Runnable() { + + @Override + public void run() { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); + } + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); + } + }); + } + } + } else { + this.jobsById.remove(job.getId()); } - return errors; + return success; } - void finishIngestJob(IngestJob job) { + synchronized void finishIngestJob(IngestJob job) { long jobId = job.getId(); this.jobsById.remove(jobId); if (!job.isCancelled()) { @@ -395,13 +475,13 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { - return !this.jobsById.isEmpty() || !ingestJobStarters.values().isEmpty(); + return !this.jobsById.isEmpty(); } /** * Cancels all ingest jobs in progress. */ - public void cancelAllIngestJobs() { + public synchronized void cancelAllIngestJobs() { // Stop creating new ingest jobs. for (Future handle : ingestJobStarters.values()) { handle.cancel(true); @@ -653,35 +733,28 @@ public class IngestManager { private final long threadId; private final IngestJob job; - private final boolean runInteractively; private ProgressHandle progress; - IngestJobStarter(long threadId, IngestJob job, boolean runInteractively) { + IngestJobStarter(long threadId, IngestJob job) { this.threadId = threadId; this.job = job; - this.runInteractively = runInteractively; } @Override public Void call() { try { if (Thread.currentThread().isInterrupted()) { + jobsById.remove(job.getId()); return null; } - /** - * Set up a progress bar. - */ if (runInteractively) { - final String displayName = NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.displayName"); + final String displayName = NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.displayName"); this.progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { if (progress != null) { - progress.setDisplayName(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.cancelling", - displayName)); + progress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.cancelling", displayName)); } Future handle = ingestJobStarters.remove(threadId); handle.cancel(true); @@ -691,35 +764,7 @@ public class IngestManager { progress.start(); } - /** - * Try to start the ingest job. - */ - List errors = IngestManager.this.startIngestJob(job, runInteractively); - if (!errors.isEmpty() && this.runInteractively) { - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); - } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - } - + startIngestJob(job); return null; } finally { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index 9ed1f33489..7e46f9b89e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -199,7 +199,7 @@ public final class RunIngestModulesDialog extends JDialog { ingestJobSettings.save(); showWarnings(ingestJobSettings); if (startIngestJob) { - IngestManager.getInstance().queueIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); + IngestManager.getInstance().queueIngestJob(RunIngestModulesDialog.this.dataSources, ingestJobSettings); } setVisible(false); dispose(); From 1f539b53b78c00b88aa630b2a3f3e829f4a459bb Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 18 Dec 2014 09:54:34 -0500 Subject: [PATCH 63/84] Add caveat to IngestManager.isIngestRunning --- Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index b0cb987710..42f518b5e4 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -470,7 +470,8 @@ public class IngestManager { } /** - * Queries whether or not any ingest jobs are in progress. + * Queries whether or not any ingest jobs are in progress, at least at the + * moment of the query. * * @return True or false. */ From 0a523552e1c06195cb5104ab7f05d6f228177292 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 18 Dec 2014 10:25:05 -0500 Subject: [PATCH 64/84] Continue work on user-defined file types --- .../sleuthkit/autopsy/coreutils/XMLUtil.java | 56 ++--- .../UserDefinedFileTypesManager.java | 192 ++++++++---------- 2 files changed, 110 insertions(+), 138 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java index 603fc52e60..bdcd28e32d 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java @@ -166,24 +166,8 @@ public class XMLUtil { } /** - * Used to consolidate more specific exception types. - */ - public static class XmlUtilException extends Exception { - - XmlUtilException(String message) { - super(message); - } - } - - /** - * Loads and validates an XML document. - * - * @param docPath The full path to the file to load. - * @param schemaPath The full path to the file to validate against. - */ - /** - * Loads and XML document and validates against a schema packaged as a - * class resource. + * Loads an XML document into a WC3 DOM and validates it against a schema + * packaged as a class resource. * * @param The name of the class associated with the resource. * @param clazz The class associated with the resource. @@ -193,21 +177,25 @@ public class XMLUtil { * @throws IOException * @throws org.sleuthkit.autopsy.coreutils.XMLUtil.XmlUtilException */ - public static Document loadAndValidateDoc(Class clazz, String docPath, String schemaResourceName) throws IOException, XmlUtilException { - try { - DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = builderFactory.newDocumentBuilder(); - Document doc = builder.parse(new FileInputStream(docPath)); - PlatformUtil.extractResourceToUserConfigDir(clazz, schemaResourceName, false); - File schemaFile = new File(Paths.get(PlatformUtil.getUserConfigDirectory(), schemaResourceName).toAbsolutePath().toString()); - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema schema = schemaFactory.newSchema(schemaFile); - Validator validator = schema.newValidator(); - validator.validate(new DOMSource(doc), new DOMResult()); - return doc; - } catch (ParserConfigurationException | SAXException ex) { - throw new XmlUtilException(ex.getLocalizedMessage()); - } + public static Document loadAndValidateDoc(Class clazz, String docPath, String schemaResourceName) throws IOException, ParserConfigurationException, SAXException { + /** + * Parse the XML file into a DOM. + */ + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document doc = builder.parse(new FileInputStream(docPath)); + + /** + * Extract the schema and validate the DOM. + */ + PlatformUtil.extractResourceToUserConfigDir(clazz, schemaResourceName, false); + File schemaFile = new File(Paths.get(PlatformUtil.getUserConfigDirectory(), schemaResourceName).toAbsolutePath().toString()); + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = schemaFactory.newSchema(schemaFile); + Validator validator = schema.newValidator(); + validator.validate(new DOMSource(doc), new DOMResult()); + + return doc; } /** @@ -220,7 +208,7 @@ public class XMLUtil { */ public static boolean saveDoc(Class clazz, String xmlPath, String encoding, final Document doc) { TransformerFactory xf = TransformerFactory.newInstance(); - xf.setAttribute("indent-number", new Integer(1)); //NON-NLS + xf.setAttribute("indent-number", 1); //NON-NLS boolean success = false; try { Transformer xformer = xf.newTransformer(); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 53ec537fa6..2b46c4880d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -5,7 +5,7 @@ * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. + * you may not use this schemaFile except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -34,15 +34,17 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import javax.xml.bind.DatatypeConverter; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.XMLUtil; import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; +import org.xml.sax.SAXException; /** - * Manages user-defined file types characterized by MIME type, signature, and - * optional membership in an interesting files set. + * Manages user-defined schemaFile types characterized by MIME type, signature, + * and optional membership in an interesting files set. */ final class UserDefinedFileTypesManager { @@ -63,34 +65,35 @@ final class UserDefinedFileTypesManager { private static UserDefinedFileTypesManager instance; /** - * Predefined file types are stored in this mapping of MIME types to file - * types. Access to this map is guarded by the intrinsic lock of the - * user-defined file types manager for thread-safety. + * Predefined schemaFile types are stored in this mapping of MIME types to + * schemaFile types. Access to this map is guarded by the intrinsic lock of + * the user-defined schemaFile types manager for thread-safety. */ private final Map predefinedFileTypes = new HashMap<>(); /** - * User-defined file types to be persisted to the user-defined file type - * definitions file are stored in this mapping of file type names to file - * types. Access to this map is guarded by the intrinsic lock of the - * user-defined file types manager for thread-safety. + * User-defined schemaFile types to be persisted to the user-defined + * schemaFile type definitions schemaFile are stored in this mapping of + * schemaFile type names to schemaFile types. Access to this map is guarded + * by the intrinsic lock of the user-defined schemaFile types manager for + * thread-safety. */ private final Map userDefinedFileTypes = new HashMap<>(); /** - * The combined set of user-defined file types and file types predefined by - * Autopsy are stored in this mapping of MIME types to file types. This is - * the current working set of file types. Access to this map is guarded by - * the intrinsic lock of the user-defined file types manager for - * thread-safety. + * The combined set of user-defined schemaFile types and schemaFile types + * predefined by Autopsy are stored in this mapping of MIME types to + * schemaFile types. This is the current working set of schemaFile types. + * Access to this map is guarded by the intrinsic lock of the user-defined + * schemaFile types manager for thread-safety. */ private final Map fileTypes = new HashMap<>(); /** - * Gets the manager of user-defined file types characterized by MIME type, - * signature, and optional membership in an interesting files set. + * Gets the manager of user-defined schemaFile types characterized by MIME + * type, signature, and optional membership in an interesting files set. * - * @return The user-defined file types manager singleton. + * @return The user-defined schemaFile types manager singleton. */ synchronized static UserDefinedFileTypesManager getInstance() { if (UserDefinedFileTypesManager.instance == null) { @@ -100,8 +103,8 @@ final class UserDefinedFileTypesManager { } /** - * Creates a manager of user-defined file types characterized by MIME type, - * signature, and optional membership in an interesting files set. + * Creates a manager of user-defined schemaFile types characterized by MIME + * type, signature, and optional membership in an interesting files set. */ private UserDefinedFileTypesManager() { /** @@ -113,11 +116,11 @@ final class UserDefinedFileTypesManager { } /** - * Adds the predefined file types to the in-memory mappings of MIME types to - * file types. + * Adds the predefined schemaFile types to the in-memory mappings of MIME + * types to schemaFile types. */ private void loadPredefinedFileTypes() { - // RJCTODO: Remove test file type. + // RJCTODO: Remove test schemaFile type. /** * Create a file type that should match $MBR in Small2 image. */ @@ -127,7 +130,7 @@ final class UserDefinedFileTypesManager { /** * Create a file type that should match test.txt in the Small2 image. */ - // RJCTODO: Remove test file type. + // RJCTODO: Remove test schemaFile type. try { fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), "predefinedASCII", true); this.addPredefinedFileType(fileType); @@ -147,7 +150,7 @@ final class UserDefinedFileTypesManager { // buf = buffer; // } // -// // the xml detection in Tika tries to parse the entire file and throws exceptions +// // the xml detection in Tika tries to parse the entire schemaFile and throws exceptions // // for files that are not valid XML // try { // String tagHeader = new String(buf, 0, 5); @@ -170,10 +173,10 @@ final class UserDefinedFileTypesManager { } /** - * Adds a file type to the the predefined file types and combined file types - * maps. + * Adds a schemaFile type to the the predefined schemaFile types and + * combined schemaFile types maps. * - * @param fileType The file type to add. + * @param fileType The schemaFile type to add. */ private void addPredefinedFileType(FileType fileType) { this.predefinedFileTypes.put(fileType.getMimeType(), fileType); @@ -181,8 +184,8 @@ final class UserDefinedFileTypesManager { } /** - * Adds the user-defined file types to the in-memory mappings of MIME types - * to file types. + * Adds the user-defined schemaFile types to the in-memory mappings of MIME + * types to schemaFile types. */ private void loadUserDefinedFileTypes() { try { @@ -194,21 +197,21 @@ final class UserDefinedFileTypesManager { } } - } catch (UserDefinedFileTypesManager.InvalidXMLException ex) { + } catch (IOException | ParserConfigurationException | SAXException ex) { /** * Using an all-or-none policy. */ UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Unable to load user-defined types", ex); //NON-NLS this.fileTypes.clear(); this.userDefinedFileTypes.clear(); - } + } } /** - * Adds a file type to the the user-defined file types and combined file - * types maps. + * Adds a schemaFile type to the the user-defined schemaFile types and + * combined schemaFile types maps. * - * @param fileType The file type to add. + * @param fileType The schemaFile type to add. */ private void addUserDefinedFileType(FileType fileType) { this.userDefinedFileTypes.put(fileType.getMimeType(), fileType); @@ -216,9 +219,10 @@ final class UserDefinedFileTypesManager { } /** - * Gets both the predefined and the user-defined file types. + * Gets both the predefined and the user-defined schemaFile types. * - * @return A mapping of file type names to file types, possibly empty. + * @return A mapping of schemaFile type names to schemaFile types, possibly + * empty. */ synchronized Map getFileTypes() { /** @@ -229,9 +233,10 @@ final class UserDefinedFileTypesManager { } /** - * Gets the user-defined file types. + * Gets the user-defined schemaFile types. * - * @return A mapping of file type names to file types, possibly empty. + * @return A mapping of schemaFile type names to schemaFile types, possibly + * empty. */ synchronized Map getUserDefinedFileTypes() { /** @@ -242,10 +247,10 @@ final class UserDefinedFileTypesManager { } /** - * Sets the user-defined file types. + * Sets the user-defined schemaFile types. * - * @param newFileTypes A mapping of file type names to user-defined file - * types. + * @param newFileTypes A mapping of schemaFile type names to user-defined + * schemaFile types. * @throws * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException */ @@ -281,10 +286,10 @@ final class UserDefinedFileTypesManager { } /** - * Gets the absolute path of a file type definitions file. + * Gets the absolute path of a schemaFile type definitions schemaFile. * - * @param fileName The name of the file. - * @return The absolute path to the file. + * @param fileName The name of the schemaFile. + * @return The absolute path to the schemaFile. */ private static String getFileTypeDefinitionsFilePath(String fileName) { Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), fileName); @@ -292,16 +297,16 @@ final class UserDefinedFileTypesManager { } /** - * Provides a mechanism for writing a set of file type definitions to an XML - * file. + * Provides a mechanism for writing a set of schemaFile type definitions to + * an XML schemaFile. */ private static class XMLWriter { /** - * Writes a set of file type definitions to an XML file. + * Writes a set of schemaFile type definitions to an XML schemaFile. * - * @param signatures A collection of file types. - * @param filePath The path to the destination file. + * @param signatures A collection of schemaFile types. + * @param filePath The path to the destination schemaFile. */ private static void writeFileTypes(Collection fileTypes, String filePath) throws ParserConfigurationException, IOException { Document doc = XMLUtil.createDoc(); @@ -318,9 +323,9 @@ final class UserDefinedFileTypesManager { } /** - * Creates an XML representation of a file type. + * Creates an XML representation of a schemaFile type. * - * @param fileType The file type object. + * @param fileType The schemaFile type object. * @param doc The DOM document to use to create the XML. * @return An XML element. */ @@ -372,33 +377,30 @@ final class UserDefinedFileTypesManager { } /** - * Provides a mechanism for reading a set of file type definitions from an - * XML file. + * Provides a mechanism for reading a set of schemaFile type definitions + * from an XML schemaFile. */ private static class XMLReader { /** - * Reads a set of file type definitions from an XML file. + * Reads a set of schemaFile type definitions from an XML schemaFile. * - * @param filePath The path to the file. - * @return A collection of file types. + * @param filePath The path to the XML schemaFile. + * @return A collection of schemaFile types read from the XML + * schemaFile. */ - private static List readFileTypes(String filePath) throws InvalidXMLException { + private static List readFileTypes(String filePath) throws IOException, ParserConfigurationException, SAXException { List fileTypes = new ArrayList<>(); - Path xsdPath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); - String xsdPathString = xsdPath.toAbsolutePath().toString(); - File file = new File(xsdPathString); - if (file.exists() && file.canRead()) { - Document doc = XMLUtil.loadAndValidateDoc(UserDefinedFileTypesManager.XMLReader.class, filePath, xsdPathString); - if (doc != null) { - Element fileTypesElem = doc.getDocumentElement(); - if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME)) { - NodeList fileTypeElems = fileTypesElem.getElementsByTagName(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); - for (int i = 0; i < fileTypeElems.getLength(); ++i) { - Element fileTypeElem = (Element) fileTypeElems.item(i); - FileType fileType = UserDefinedFileTypesManager.XMLReader.parseFileType(fileTypeElem); - fileTypes.add(fileType); - } + Path schemaFilePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); + Document doc = XMLUtil.loadAndValidateDoc(UserDefinedFileTypesManager.XMLReader.class, filePath, schemaFilePath.toAbsolutePath().toString()); + if (doc != null) { + Element fileTypesElem = doc.getDocumentElement(); + if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME)) { + NodeList fileTypeElems = fileTypesElem.getElementsByTagName(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); + for (int i = 0; i < fileTypeElems.getLength(); ++i) { + Element fileTypeElem = (Element) fileTypeElems.item(i); + FileType fileType = UserDefinedFileTypesManager.XMLReader.parseFileType(fileTypeElem); + fileTypes.add(fileType); } } } @@ -406,29 +408,23 @@ final class UserDefinedFileTypesManager { } /** - * Parse a file type definition from a file type XML element. + * Parse a schemaFile type definition from a schemaFile type XML + * element. * * @param fileTypeElem The XML element. - * @return A file type object. + * @return A schemaFile type object. * @throws - * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.InvalidXMLException */ - private static FileType parseFileType(Element fileTypeElem) throws InvalidXMLException, IllegalArgumentException, NumberFormatException { + private static FileType parseFileType(Element fileTypeElem) throws IllegalArgumentException, NumberFormatException { /** * Get the mime type child element. */ String mimeType = UserDefinedFileTypesManager.getChildElementTextContent(fileTypeElem, UserDefinedFileTypesManager.MIME_TYPE_TAG_NAME); /** - * Get the signature child element. The check here is essentially a - * "sanity check" since the XML was already validated using the XSD - * file. + * Get the signature child element. */ NodeList signatureElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypesManager.SIGNATURE_TAG_NAME); - if (signatureElems.getLength() < 1) { - } else { - throw new InvalidXMLException("Missing " + UserDefinedFileTypesManager.SIGNATURE_TAG_NAME + " child element"); //NON-NLS - } Element signatureElem = (Element) signatureElems.item(0); /** @@ -472,35 +468,22 @@ final class UserDefinedFileTypesManager { } /** - * Gets the text content of a single child element. + * Gets the text content of a single child element. Assumes the elements + * have already been validated. * * @param elem The parent element. * @param tagName The tag name of the child element. * @return The text content. - * @throws UserDefinedFileTypesException */ - private static String getChildElementTextContent(Element elem, String tagName) throws InvalidXMLException { - /** - * The checks here are essentially "sanity checks" since the XML was - * already validated using the XSD file. - */ + private static String getChildElementTextContent(Element elem, String tagName) { NodeList childElems = elem.getElementsByTagName(tagName); - if (childElems.getLength() > 0) { - Element childElem = (Element) childElems.item(0); - String textContent = childElem.getTextContent(); - if (!textContent.isEmpty()) { - return textContent; - } else { - throw new InvalidXMLException(tagName + " child element missing text content"); //NON-NLS - } - } else { - throw new InvalidXMLException("Missing " + tagName + " child element"); //NON-NLS - } + Element childElem = (Element) childElems.item(0); + return childElem.getTextContent(); } /** - * Used for exceptions when parsing user-defined types XML elements and - * attributes. + * Used for throwing exceptions when parsing user-defined types XML elements + * and attributes. */ private static class InvalidXMLException extends Exception { @@ -511,7 +494,8 @@ final class UserDefinedFileTypesManager { /** * Used to translate more implementation-details-specific exceptions (which - * are logged by this class) into more generic exceptions. + * are logged by this class) into more generic exceptions for propagation to + * clients of the user-defined schemaFile types manager. */ static class UserDefinedFileTypesException extends Exception { From 56e83b59d05f8f22278a943ec755e028912f26c4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 18 Dec 2014 12:45:43 -0500 Subject: [PATCH 65/84] Fix incorrect null check in IngestJob.getSnapshot --- Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index d6725b8220..533a4f842a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -111,7 +111,7 @@ public final class IngestJob { boolean fileIngestIsRunning = false; Date fileIngestStartTime = null; for (DataSourceIngestJob.Snapshot snapshot : this.getDataSourceIngestJobSnapshots()) { - if (null != moduleHandle) { + if (null == moduleHandle) { DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); if (null != module) { moduleHandle = new DataSourceIngestModuleHandle(this.dataSourceJobs.get(snapshot.getJobId()), module); From 1eef4f10c6755c7255784d7af46aa05faa65118f Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 18 Dec 2014 13:04:15 -0500 Subject: [PATCH 66/84] Add isCancelled to IngestJob.DataSourceIngestModuleHandle --- .../org/sleuthkit/autopsy/ingest/IngestJob.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 533a4f842a..fe129c3528 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -257,6 +257,7 @@ public final class IngestJob { private final DataSourceIngestJob job; private final DataSourceIngestPipeline.PipelineModule module; + private final boolean cancelled; /** * Constructs a handle to a data source level ingest module that can be @@ -270,6 +271,7 @@ public final class IngestJob { private DataSourceIngestModuleHandle(DataSourceIngestJob job, DataSourceIngestPipeline.PipelineModule module) { this.job = job; this.module = module; + this.cancelled = job.currentDataSourceIngestModuleIsCancelled(); } /** @@ -283,7 +285,7 @@ public final class IngestJob { } /** - * Returns the time the data source level ingest module associated with + * Gets the time the data source level ingest module associated with * this handle began processing. * * @return The module processing start time. @@ -292,6 +294,16 @@ public final class IngestJob { return this.module.getProcessingStartTime(); } + /** + * Queries whether or not cancellation of the data source level ingest + * module associated with this handle has been requested. + * + * @return True or false. + */ + public boolean isCancelled() { + return this.cancelled; + } + /** * Requests cancellation of the ingest module associated with this * handle. Returns immediately, but there may be a delay before the From dcd632e5e6193d81fc226b15b5caaa9fc4ee3f2c Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 18 Dec 2014 16:59:34 -0500 Subject: [PATCH 67/84] Continue to work on user-defined file types --- .../sleuthkit/autopsy/coreutils/XMLUtil.java | 124 ++++-- .../autopsy/modules/filetypeid/FileType.java | 7 +- .../FileTypeIdGlobalSettingsPanel.java | 58 +-- .../filetypeid/UserDefinedFileTypes.xsd | 12 +- .../UserDefinedFileTypesManager.java | 364 ++++++++++-------- 5 files changed, 304 insertions(+), 261 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java index bdcd28e32d..ef360762d9 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java @@ -56,17 +56,100 @@ import org.xml.sax.SAXException; public class XMLUtil { /** - * Creates a W3C DOM document. + * Creates a W3C DOM. * * @return The document object. * @throws ParserConfigurationException */ - public static Document createDoc() throws ParserConfigurationException { + public static Document createDocument() throws ParserConfigurationException { DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = builderFactory.newDocumentBuilder(); return builder.newDocument(); } + /** + * Loads an XML document into a WC3 DOM and validates it using a schema + * packaged as a class resource. + * + * @param The name of the class associated with the resource. + * @param docPath The full path to the XML document. + * @param clazz The class associated with the schema resource. + * @param schemaResourceName The name of the schema resource. + * @return The WC3 DOM document object. + * @throws IOException + * @throws ParserConfigurationException + * @throws SAXException + */ + public static Document loadDocument(String docPath, Class clazz, String schemaResourceName) throws IOException, ParserConfigurationException, SAXException { + Document doc = loadDocument(docPath); + validateDocument(doc, clazz, schemaResourceName); + return doc; + } + + /** + * Loads an XML document into a WC3 DOM. + * + * @param docPath The full path to the XML document. + * @return The WC3 DOM document object. + * @throws ParserConfigurationException + * @throws SAXException + * @throws IOException + */ + public static Document loadDocument(String docPath) throws ParserConfigurationException, SAXException, IOException { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document doc = builder.parse(new FileInputStream(docPath)); + return doc; + } + + /** + * Validates a WC3 DOM using a schema packaged as a class resource. + * + * @param doc + * @param clazz + * @param schemaResourceName + * @throws SAXException + * @throws IOException + */ + public static void validateDocument(final Document doc, Class clazz, String schemaResourceName) throws SAXException, IOException { + PlatformUtil.extractResourceToUserConfigDir(clazz, schemaResourceName, false); + File schemaFile = new File(Paths.get(PlatformUtil.getUserConfigDirectory(), schemaResourceName).toAbsolutePath().toString()); + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = schemaFactory.newSchema(schemaFile); + Validator validator = schema.newValidator(); + validator.validate(new DOMSource(doc), new DOMResult()); + } + + /** + * Saves a WC3 DOM by writing it to an XML document. + * + * @param doc The WC3 DOM document object. + * @param docPath The full path to the XML document. + * @param encoding Encoding scheme to use for the XML document, e.g., + * "UTF-8." + * @throws TransformerConfigurationException + * @throws FileNotFoundException + * @throws UnsupportedEncodingException + * @throws TransformerException + * @throws IOException + */ + public static void saveDocument(final Document doc, String encoding, String docPath) throws TransformerConfigurationException, FileNotFoundException, UnsupportedEncodingException, TransformerException, IOException { + TransformerFactory xf = TransformerFactory.newInstance(); + xf.setAttribute("indent-number", 1); //NON-NLS + Transformer xformer = xf.newTransformer(); + xformer.setOutputProperty(OutputKeys.METHOD, "xml"); //NON-NLS + xformer.setOutputProperty(OutputKeys.INDENT, "yes"); //NON-NLS + xformer.setOutputProperty(OutputKeys.ENCODING, encoding); + xformer.setOutputProperty(OutputKeys.STANDALONE, "yes"); //NON-NLS + xformer.setOutputProperty(OutputKeys.VERSION, "1.0"); + File file = new File(docPath); + try (FileOutputStream stream = new FileOutputStream(file)) { + Result out = new StreamResult(new OutputStreamWriter(stream, encoding)); + xformer.transform(new DOMSource(doc), out); + stream.flush(); + } + } + /** * Utility to validate XML files against pre-defined schema files. * @@ -84,6 +167,7 @@ public class XMLUtil { * IngestModuleLoader. * */ + // RJCTODO: Deprecate. public static boolean xmlIsValid(DOMSource xmlfile, Class clazz, String schemaFile) { try { PlatformUtil.extractResourceToUserConfigDir(clazz, schemaFile, false); @@ -121,6 +205,7 @@ public class XMLUtil { * IngestModuleLoader. * */ + // RJCTODO: Deprecate. public static boolean xmlIsValid(Document doc, Class clazz, String type) { DOMSource dms = new DOMSource(doc); return xmlIsValid(dms, clazz, type); @@ -132,6 +217,7 @@ public class XMLUtil { * @param clazz the class this method is invoked from * @param xmlPath the full path to the file to load */ + // RJCTODO: Deprecate. public static Document loadDoc(Class clazz, String xmlPath) { DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); Document ret = null; @@ -165,39 +251,6 @@ public class XMLUtil { return ret; } - /** - * Loads an XML document into a WC3 DOM and validates it against a schema - * packaged as a class resource. - * - * @param The name of the class associated with the resource. - * @param clazz The class associated with the resource. - * @param docPath The full path to the XML document. - * @param schemaResourceName The name of the schema resource - * @return A WC3 DOM representation of the document - * @throws IOException - * @throws org.sleuthkit.autopsy.coreutils.XMLUtil.XmlUtilException - */ - public static Document loadAndValidateDoc(Class clazz, String docPath, String schemaResourceName) throws IOException, ParserConfigurationException, SAXException { - /** - * Parse the XML file into a DOM. - */ - DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = builderFactory.newDocumentBuilder(); - Document doc = builder.parse(new FileInputStream(docPath)); - - /** - * Extract the schema and validate the DOM. - */ - PlatformUtil.extractResourceToUserConfigDir(clazz, schemaResourceName, false); - File schemaFile = new File(Paths.get(PlatformUtil.getUserConfigDirectory(), schemaResourceName).toAbsolutePath().toString()); - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema schema = schemaFactory.newSchema(schemaFile); - Validator validator = schema.newValidator(); - validator.validate(new DOMSource(doc), new DOMResult()); - - return doc; - } - /** * Saves XML files to disk * @@ -206,6 +259,7 @@ public class XMLUtil { * @param encoding to encoding, such as "UTF-8", to encode the file with * @param doc the document to save */ + // RJCTODO: Deprecate. public static boolean saveDoc(Class clazz, String xmlPath, String encoding, final Document doc) { TransformerFactory xf = TransformerFactory.newInstance(); xf.setAttribute("indent-number", 1); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java index e3b9908131..55639114ec 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -88,10 +88,11 @@ final class FileType { boolean alertOnMatch() { return this.alert; } - + /** - * Gets the interesting files set name assigned to this file type - * @return + * Gets the interesting files set name assigned to this file type. + * + * @return The files set name, possibly empty. */ String getFilesSetName() { return this.filesSetName; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 70aebae4d5..178937ca56 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -44,19 +44,10 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private static final String RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM = NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureComboBox.rawItem"); private static final String ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM = NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureComboBox.asciiItem"); - /** - * These two fields are used to synthesize default names for user-defined - * types. This is a thread-safe implementation. All interactions with - * instances of this panel should occur on the EDT, so this is defensive - * programming. - */ - private static final String DEFAULT_TYPE_NAME_BASE = "userdefined/userdefined"; //NON-NLS - private static final AtomicInteger defaultTypeNameCounter = new AtomicInteger(1); // RJCTODO: Need to save and init counter - /** * The list model for the file types list component of this panel is the set * of type names of the user-defined file types. A mapping of the file type - * names to file type objects completes the model. + * names to file type objects lies behind the list model. */ private DefaultListModel typesListModel; private Map fileTypes; @@ -70,24 +61,18 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } /** - * Does child component initialization in addition to the the Matisse - * generated initialization. + * Does child component initialization in addition to that done by the + * Matisse generated code. */ private void customizeComponents() { - /** - * Make a model for the file types list component. - */ this.typesListModel = new DefaultListModel<>(); this.typesList.setModel(this.typesListModel); - /** - * Make a model for the signature type combo box component. - */ DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); - + this.filesSetNameTextField.setEnabled(false); } @@ -100,35 +85,20 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } /** - * Populates the child components with file types obtained from the - * user-defined file types manager. + * @inheritDoc */ @Override public void load() { - /** - * Get the user-defined file types and set up a list model for the file - * types list component. - */ this.fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); this.setFileTypesListModel(); - - /** - * Add a selection listener to populate the file type details - * display/edit components. - */ this.typesList.addListSelectionListener(new TypesListSelectionListener()); - - /** - * If there is at least one user-defined file type, select it the file - * types list component. - */ if (!this.typesListModel.isEmpty()) { this.typesList.setSelectedIndex(0); } } /** - * Stores any changes to the user-defined types. + * @inheritDoc */ @Override public void store() { @@ -158,6 +128,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane FileTypeIdGlobalSettingsPanel.this.signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); FileTypeIdGlobalSettingsPanel.this.offsetTextField.setText(Long.toString(signature.getOffset())); FileTypeIdGlobalSettingsPanel.this.postHitCheckBox.setSelected(fileType.alertOnMatch()); + FileTypeIdGlobalSettingsPanel.this.filesSetNameTextField.setText(fileType.getFilesSetName()); FileTypeIdGlobalSettingsPanel.this.deleteTypeButton.setEnabled(true); } } @@ -187,6 +158,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane this.signatureTextField.setText(""); //NON-NLS this.offsetTextField.setText(""); //NON-NLS this.postHitCheckBox.setSelected(false); + this.filesSetNameTextField.setText(""); } /** @@ -372,7 +344,6 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private void newTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_newTypeButtonActionPerformed this.clearTypeDetailsComponents(); - this.mimeTypeTextField.setText(FileTypeIdGlobalSettingsPanel.DEFAULT_TYPE_NAME_BASE + FileTypeIdGlobalSettingsPanel.defaultTypeNameCounter.getAndIncrement()); }//GEN-LAST:event_newTypeButtonActionPerformed private void deleteTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteTypeButtonActionPerformed @@ -429,16 +400,16 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane */ String filesSetName = this.filesSetNameTextField.getText(); if (this.postHitCheckBox.isSelected() && filesSetName.isEmpty()) { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), - JOptionPane.ERROR_MESSAGE); + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), + JOptionPane.ERROR_MESSAGE); } - + /** * Put it all together and reset the file types list component. */ - FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); // RJCTODO: + FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); FileType fileType = new FileType(typeName, signature, filesSetName, this.postHitCheckBox.isSelected()); this.fileTypes.put(typeName, fileType); this.setFileTypesListModel(); @@ -461,7 +432,6 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane this.filesSetNameTextField.setEnabled(this.postHitCheckBox.isSelected()); }//GEN-LAST:event_postHitCheckBoxActionPerformed - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton deleteTypeButton; private javax.swing.JLabel filesSetNameLabel; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd index 16557dc422..47f658599a 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd @@ -26,14 +26,7 @@ - - - - - - - - + @@ -41,7 +34,8 @@ - + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 2b46c4880d..65a7a447a2 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -5,7 +5,7 @@ * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this schemaFile except in compliance with the License. + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.file.Path; @@ -34,17 +35,16 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import javax.xml.bind.DatatypeConverter; -import org.openide.util.Exceptions; +import javax.xml.transform.TransformerException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.XMLUtil; import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; -import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; import org.xml.sax.SAXException; /** - * Manages user-defined schemaFile types characterized by MIME type, signature, - * and optional membership in an interesting files set. + * Manages user-defined file types characterized by MIME type, signature, and + * optional membership in an interesting files set. */ final class UserDefinedFileTypesManager { @@ -65,35 +65,34 @@ final class UserDefinedFileTypesManager { private static UserDefinedFileTypesManager instance; /** - * Predefined schemaFile types are stored in this mapping of MIME types to - * schemaFile types. Access to this map is guarded by the intrinsic lock of - * the user-defined schemaFile types manager for thread-safety. + * Predefined file types are stored in this mapping of MIME types to file + * types. Access to this map is guarded by the intrinsic lock of the + * user-defined file types manager for thread-safety. */ private final Map predefinedFileTypes = new HashMap<>(); /** - * User-defined schemaFile types to be persisted to the user-defined - * schemaFile type definitions schemaFile are stored in this mapping of - * schemaFile type names to schemaFile types. Access to this map is guarded - * by the intrinsic lock of the user-defined schemaFile types manager for - * thread-safety. + * File types to be persisted to the user-defined file type definitions file + * are stored in this mapping of MIME types to file types. Access to this + * map is guarded by the intrinsic lock of the user-defined file types + * manager for thread-safety. */ private final Map userDefinedFileTypes = new HashMap<>(); /** - * The combined set of user-defined schemaFile types and schemaFile types - * predefined by Autopsy are stored in this mapping of MIME types to - * schemaFile types. This is the current working set of schemaFile types. - * Access to this map is guarded by the intrinsic lock of the user-defined - * schemaFile types manager for thread-safety. + * The combined set of user-defined file types and file types predefined by + * Autopsy are stored in this mapping of MIME types to file types. This is + * the current working set of file types. Access to this map is guarded by + * the intrinsic lock of the user-defined file types manager for + * thread-safety. */ private final Map fileTypes = new HashMap<>(); - + /** - * Gets the manager of user-defined schemaFile types characterized by MIME - * type, signature, and optional membership in an interesting files set. + * Gets the manager of user-defined file types characterized by MIME type, + * signature, and optional membership in an interesting files set. * - * @return The user-defined schemaFile types manager singleton. + * @return The user-defined file types manager singleton. */ synchronized static UserDefinedFileTypesManager getInstance() { if (UserDefinedFileTypesManager.instance == null) { @@ -103,8 +102,8 @@ final class UserDefinedFileTypesManager { } /** - * Creates a manager of user-defined schemaFile types characterized by MIME - * type, signature, and optional membership in an interesting files set. + * Creates a manager of user-defined file types characterized by MIME type, + * signature, and optional membership in an interesting files set. */ private UserDefinedFileTypesManager() { /** @@ -116,11 +115,11 @@ final class UserDefinedFileTypesManager { } /** - * Adds the predefined schemaFile types to the in-memory mappings of MIME - * types to schemaFile types. + * Adds the predefined file types to the in-memory mappings of MIME types to + * file types. */ private void loadPredefinedFileTypes() { - // RJCTODO: Remove test schemaFile type. + // RJCTODO: Remove test file type. /** * Create a file type that should match $MBR in Small2 image. */ @@ -130,7 +129,7 @@ final class UserDefinedFileTypesManager { /** * Create a file type that should match test.txt in the Small2 image. */ - // RJCTODO: Remove test schemaFile type. + // RJCTODO: Remove test file type. try { fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), "predefinedASCII", true); this.addPredefinedFileType(fileType); @@ -150,7 +149,7 @@ final class UserDefinedFileTypesManager { // buf = buffer; // } // -// // the xml detection in Tika tries to parse the entire schemaFile and throws exceptions +// // the xml detection in Tika tries to parse the entire file and throws exceptions // // for files that are not valid XML // try { // String tagHeader = new String(buf, 0, 5); @@ -173,10 +172,10 @@ final class UserDefinedFileTypesManager { } /** - * Adds a schemaFile type to the the predefined schemaFile types and - * combined schemaFile types maps. + * Adds a predefined file type to the in-memory mappings of MIME types to + * file types. * - * @param fileType The schemaFile type to add. + * @param fileType The file type to add. */ private void addPredefinedFileType(FileType fileType) { this.predefinedFileTypes.put(fileType.getMimeType(), fileType); @@ -184,15 +183,15 @@ final class UserDefinedFileTypesManager { } /** - * Adds the user-defined schemaFile types to the in-memory mappings of MIME - * types to schemaFile types. + * Adds the user-defined file types to the in-memory mappings of MIME types + * to file types. */ private void loadUserDefinedFileTypes() { try { String filePath = getFileTypeDefinitionsFilePath(UserDefinedFileTypesManager.USER_DEFINED_TYPE_DEFINITIONS_FILE); File file = new File(filePath); if (file.exists() && file.canRead()) { - for (FileType fileType : XMLReader.readFileTypes(filePath)) { + for (FileType fileType : XmlReader.readFileTypes(filePath)) { this.addUserDefinedFileType(fileType); } } @@ -204,14 +203,14 @@ final class UserDefinedFileTypesManager { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Unable to load user-defined types", ex); //NON-NLS this.fileTypes.clear(); this.userDefinedFileTypes.clear(); - } + } } /** - * Adds a schemaFile type to the the user-defined schemaFile types and - * combined schemaFile types maps. + * Adds a user-defined file type to the in-memory mappings of MIME types to + * file types. * - * @param fileType The schemaFile type to add. + * @param fileType The file type to add. */ private void addUserDefinedFileType(FileType fileType) { this.userDefinedFileTypes.put(fileType.getMimeType(), fileType); @@ -219,10 +218,9 @@ final class UserDefinedFileTypesManager { } /** - * Gets both the predefined and the user-defined schemaFile types. + * Gets both the predefined and the user-defined file types. * - * @return A mapping of schemaFile type names to schemaFile types, possibly - * empty. + * @return A mapping of file type names to file types, possibly empty. */ synchronized Map getFileTypes() { /** @@ -233,10 +231,9 @@ final class UserDefinedFileTypesManager { } /** - * Gets the user-defined schemaFile types. + * Gets the user-defined file types. * - * @return A mapping of schemaFile type names to schemaFile types, possibly - * empty. + * @return A mapping of file type names to file types, possibly empty. */ synchronized Map getUserDefinedFileTypes() { /** @@ -247,22 +244,20 @@ final class UserDefinedFileTypesManager { } /** - * Sets the user-defined schemaFile types. + * Sets the user-defined file types. * - * @param newFileTypes A mapping of schemaFile type names to user-defined - * schemaFile types. - * @throws - * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + * @param newFileTypes A mapping of file type names to user-defined file + * types. */ synchronized void setUserDefinedFileTypes(Map newFileTypes) throws UserDefinedFileTypesManager.UserDefinedFileTypesException { try { - /** - * Persist the user-defined file type definitions. - */ String filePath = UserDefinedFileTypesManager.getFileTypeDefinitionsFilePath(UserDefinedFileTypesManager.USER_DEFINED_TYPE_DEFINITIONS_FILE); - UserDefinedFileTypesManager.XMLWriter.writeFileTypes(newFileTypes.values(), filePath); + UserDefinedFileTypesManager.XmlWriter.writeFileTypes(newFileTypes.values(), filePath); - } catch (ParserConfigurationException | IOException ex) { + } catch (ParserConfigurationException | FileNotFoundException | UnsupportedEncodingException | TransformerException ex) { + UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); + throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message + } catch (IOException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message } @@ -286,10 +281,10 @@ final class UserDefinedFileTypesManager { } /** - * Gets the absolute path of a schemaFile type definitions schemaFile. + * Gets the absolute path of a file type definitions file. * - * @param fileName The name of the schemaFile. - * @return The absolute path to the schemaFile. + * @param fileName The name of the file. + * @return The absolute path to the file. */ private static String getFileTypeDefinitionsFilePath(String fileName) { Path filePath = Paths.get(PlatformUtil.getUserConfigDirectory(), fileName); @@ -297,109 +292,133 @@ final class UserDefinedFileTypesManager { } /** - * Provides a mechanism for writing a set of schemaFile type definitions to - * an XML schemaFile. + * Provides a mechanism for writing a set of file type definitions to an XML + * file. */ - private static class XMLWriter { + private static class XmlWriter { /** - * Writes a set of schemaFile type definitions to an XML schemaFile. + * Writes a set of file type definitions to an XML file. * - * @param signatures A collection of schemaFile types. - * @param filePath The path to the destination schemaFile. + * @param fileTypes A collection of file types. + * @param filePath The path to the destination file. + * @throws ParserConfigurationException + * @throws IOException + * @throws FileNotFoundException + * @throws UnsupportedEncodingException + * @throws TransformerException */ - private static void writeFileTypes(Collection fileTypes, String filePath) throws ParserConfigurationException, IOException { - Document doc = XMLUtil.createDoc(); + private static void writeFileTypes(Collection fileTypes, String filePath) throws ParserConfigurationException, IOException, FileNotFoundException, UnsupportedEncodingException, TransformerException { + Document doc = XMLUtil.createDocument(); Element fileTypesElem = doc.createElement(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME); doc.appendChild(fileTypesElem); for (FileType fileType : fileTypes) { - Element fileTypeElem = UserDefinedFileTypesManager.XMLWriter.createFileTypeElement(fileType, doc); + Element fileTypeElem = UserDefinedFileTypesManager.XmlWriter.createFileTypeElement(fileType, doc); fileTypesElem.appendChild(fileTypeElem); } - if (!XMLUtil.saveDoc(HashDbManager.class, filePath, UserDefinedFileTypesManager.ENCODING_FOR_XML_FILE, doc)) { - // RJCTODO: If time permits add XMLUtil that properly throws and deprecate this one - throw new IOException("Error saving user defined file types, see log for details"); //NON-NLS - } + XMLUtil.saveDocument(doc, UserDefinedFileTypesManager.ENCODING_FOR_XML_FILE, filePath); } /** - * Creates an XML representation of a schemaFile type. + * Creates an XML representation of a file type. * - * @param fileType The schemaFile type object. - * @param doc The DOM document to use to create the XML. + * @param fileType The file type object. + * @param doc The WC3 DOM object to use to create the XML. * @return An XML element. */ private static Element createFileTypeElement(FileType fileType, Document doc) { - /** - * Create a file type element. - */ Element fileTypeElem = doc.createElement(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); + XmlWriter.addMimeTypeElement(fileType, fileTypeElem, doc); + XmlWriter.addSignatureElement(fileType, fileTypeElem, doc); + XmlWriter.addInterestingFilesSetElement(fileType, fileTypeElem, doc); + XmlWriter.addAlertAttribute(fileType, fileTypeElem); + return fileTypeElem; + } - /** - * Add a MIME type name child element. - */ + /** + * Add a MIME type child element to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + * @param doc The WC3 DOM object to use to create the XML. + */ + private static void addMimeTypeElement(FileType fileType, Element fileTypeElem, Document doc) { Element typeNameElem = doc.createElement(UserDefinedFileTypesManager.MIME_TYPE_TAG_NAME); typeNameElem.setTextContent(fileType.getMimeType()); fileTypeElem.appendChild(typeNameElem); + } - /** - * Add a signature child element with a type attribute. - */ + /** + * Add a signature child element to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + * @param doc The WC3 DOM object to use to create the XML. + */ + private static void addSignatureElement(FileType fileType, Element fileTypeElem, Document doc) { Signature signature = fileType.getSignature(); Element signatureElem = doc.createElement(UserDefinedFileTypesManager.SIGNATURE_TAG_NAME); - signatureElem.setAttribute(UserDefinedFileTypesManager.SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); - fileTypeElem.appendChild(signatureElem); - /** - * Add a bytes child element to the signature element. - */ Element bytesElem = doc.createElement(UserDefinedFileTypesManager.BYTES_TAG_NAME); bytesElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); signatureElem.appendChild(bytesElem); - /** - * Add an offset child element to the signature element. - */ Element offsetElem = doc.createElement(UserDefinedFileTypesManager.OFFSET_TAG_NAME); offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); signatureElem.appendChild(offsetElem); - /** - * Add a files set child element with an alert attribute. - */ + signatureElem.setAttribute(UserDefinedFileTypesManager.SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); + fileTypeElem.appendChild(signatureElem); + } + + /** + * Add an interesting files set element to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + * @param doc The WC3 DOM object to use to create the XML. + */ + private static void addInterestingFilesSetElement(FileType fileType, Element fileTypeElem, Document doc) { Element filesSetElem = doc.createElement(UserDefinedFileTypesManager.INTERESTING_FILES_SET_TAG_NAME); filesSetElem.setTextContent(fileType.getFilesSetName()); - filesSetElem.setAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); fileTypeElem.appendChild(filesSetElem); - - return fileTypeElem; } + + /** + * Add an alert attribute to a file type XML element. + * + * @param fileType The file type to use as a content source. + * @param fileTypeElem The parent file type element. + */ + private static void addAlertAttribute(FileType fileType, Element fileTypeElem) { + fileTypeElem.setAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); + } + } /** - * Provides a mechanism for reading a set of schemaFile type definitions - * from an XML schemaFile. + * Provides a mechanism for reading a set of file type definitions from an + * XML file. */ - private static class XMLReader { + private static class XmlReader { /** - * Reads a set of schemaFile type definitions from an XML schemaFile. + * Reads a set of file type definitions from an XML file. * - * @param filePath The path to the XML schemaFile. - * @return A collection of schemaFile types read from the XML - * schemaFile. + * @param filePath The path to the XML file. + * @return A collection of file types read from the XML file. */ private static List readFileTypes(String filePath) throws IOException, ParserConfigurationException, SAXException { List fileTypes = new ArrayList<>(); Path schemaFilePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); - Document doc = XMLUtil.loadAndValidateDoc(UserDefinedFileTypesManager.XMLReader.class, filePath, schemaFilePath.toAbsolutePath().toString()); + Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.XmlReader.class, schemaFilePath.toAbsolutePath().toString()); if (doc != null) { Element fileTypesElem = doc.getDocumentElement(); if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME)) { NodeList fileTypeElems = fileTypesElem.getElementsByTagName(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); for (int i = 0; i < fileTypeElems.getLength(); ++i) { Element fileTypeElem = (Element) fileTypeElems.item(i); - FileType fileType = UserDefinedFileTypesManager.XMLReader.parseFileType(fileTypeElem); + FileType fileType = XmlReader.parseFileType(fileTypeElem); fileTypes.add(fileType); } } @@ -408,94 +427,99 @@ final class UserDefinedFileTypesManager { } /** - * Parse a schemaFile type definition from a schemaFile type XML - * element. + * Gets a file type definition from a file type XML element. * * @param fileTypeElem The XML element. - * @return A schemaFile type object. - * @throws + * @return A file type object. + * @throws IllegalArgumentException + * @throws NumberFormatException */ private static FileType parseFileType(Element fileTypeElem) throws IllegalArgumentException, NumberFormatException { - /** - * Get the mime type child element. - */ - String mimeType = UserDefinedFileTypesManager.getChildElementTextContent(fileTypeElem, UserDefinedFileTypesManager.MIME_TYPE_TAG_NAME); + String mimeType = XmlReader.parseMimeType(fileTypeElem); + Signature signature = XmlReader.parseSignature(fileTypeElem); + String filesSetName = XmlReader.parseInterestingFilesSet(fileTypeElem); + boolean alert = XmlReader.parseAlert(fileTypeElem); + return new FileType(mimeType, signature, filesSetName, alert); + } - /** - * Get the signature child element. - */ + /** + * Gets the MIME type from a file type XML element. + * + * @param fileTypeElem The element + * @return A MIME type string. + */ + private static String parseMimeType(Element fileTypeElem) { + return getChildElementTextContent(fileTypeElem, UserDefinedFileTypesManager.MIME_TYPE_TAG_NAME); + } + + /** + * Gets the signature from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return The signature. + */ + private static Signature parseSignature(Element fileTypeElem) throws IllegalArgumentException, NumberFormatException { NodeList signatureElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypesManager.SIGNATURE_TAG_NAME); Element signatureElem = (Element) signatureElems.item(0); - /** - * Get the signature (interpretation) type attribute from the - * signature child element. - */ String sigTypeAttribute = signatureElem.getAttribute(UserDefinedFileTypesManager.SIGNATURE_TYPE_ATTRIBUTE); Signature.Type signatureType = Signature.Type.valueOf(sigTypeAttribute); - /** - * Get the signature bytes. - */ - String sigBytesString = UserDefinedFileTypesManager.getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.BYTES_TAG_NAME); + String sigBytesString = getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.BYTES_TAG_NAME); byte[] signatureBytes = DatatypeConverter.parseHexBinary(sigBytesString); - /** - * Get the offset. - */ - String offsetString = UserDefinedFileTypesManager.getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.OFFSET_TAG_NAME); + String offsetString = getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.OFFSET_TAG_NAME); long offset = DatatypeConverter.parseLong(offsetString); - /** - * Get the interesting files set element. - */ + return new Signature(signatureBytes, offset, signatureType); + } + + /** + * Gets the interesting files set name from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return The files set name, possibly empty. + */ + private static String parseInterestingFilesSet(Element fileTypeElem) { + String filesSetName = ""; NodeList filesSetElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypesManager.INTERESTING_FILES_SET_TAG_NAME); - Element filesSetElem = (Element) filesSetElems.item(0); - String filesSetName = filesSetElem.getTextContent(); - - /** - * Get the alert attribute from the interesting files set element. - */ - String alertAttribute = filesSetElem.getAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE); - boolean alert = Boolean.parseBoolean(alertAttribute); - - /** - * Put it all together. - */ - Signature signature = new Signature(signatureBytes, offset, signatureType); - return new FileType(mimeType, signature, filesSetName, alert); + if (filesSetElems.getLength() > 0) { + Element filesSetElem = (Element) filesSetElems.item(0); + filesSetName = filesSetElem.getTextContent(); + } + return filesSetName; } - } - /** - * Gets the text content of a single child element. Assumes the elements - * have already been validated. - * - * @param elem The parent element. - * @param tagName The tag name of the child element. - * @return The text content. - */ - private static String getChildElementTextContent(Element elem, String tagName) { - NodeList childElems = elem.getElementsByTagName(tagName); - Element childElem = (Element) childElems.item(0); - return childElem.getTextContent(); - } - - /** - * Used for throwing exceptions when parsing user-defined types XML elements - * and attributes. - */ - private static class InvalidXMLException extends Exception { - - InvalidXMLException(String message) { - super(message); + /** + * Gets the alert attribute from a file type XML element. + * + * @param fileTypeElem The XML element. + * @return True or false; + */ + private static boolean parseAlert(Element fileTypeElem) { + String alertAttribute = fileTypeElem.getAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE); + return Boolean.parseBoolean(alertAttribute); } + + /** + * Gets the text content of a single child element. + * + * @param elem The parent element. + * @param tagName The tag name of the child element. + * @return The text content. + */ + private static String getChildElementTextContent(Element elem, String tagName) { + NodeList childElems = elem.getElementsByTagName(tagName); + Element childElem = (Element) childElems.item(0); + return childElem.getTextContent(); + } + } /** * Used to translate more implementation-details-specific exceptions (which * are logged by this class) into more generic exceptions for propagation to - * clients of the user-defined schemaFile types manager. + * clients of the user-defined file types manager. */ static class UserDefinedFileTypesException extends Exception { From 86e9ba0692476cff5657a92d5722ffbf12f53c56 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 18 Dec 2014 22:45:34 -0500 Subject: [PATCH 68/84] Continue to work on user-defined file types --- .../modules/filetypeid/Bundle.properties | 1 + .../FileTypeIdGlobalSettingsPanel.java | 37 +++++++++++++------ .../filetypeid/FileTypeIdIngestModule.java | 34 ++++++++++++----- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index 2f55fc8fb8..e81c0f9e02 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -33,3 +33,4 @@ FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message=The sign FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Signature FileTypeIdGlobalSettingsPanel.filesSetNameLabel.text=Files Set Name FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text= +FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title=Save Failed diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 178937ca56..754d52b888 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -22,7 +22,6 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; import javax.swing.DefaultComboBoxModel; import javax.swing.DefaultListModel; import javax.swing.JOptionPane; @@ -46,8 +45,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * The list model for the file types list component of this panel is the set - * of type names of the user-defined file types. A mapping of the file type - * names to file type objects lies behind the list model. + * of MIME types associated with the user-defined file types. A mapping of + * the MIME types to file type objects lies behind the list model. This map + * is obtained from the user-defined types manager. */ private DefaultListModel typesListModel; private Map fileTypes; @@ -73,6 +73,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); + this.postHitCheckBox.setSelected(false); this.filesSetNameTextField.setEnabled(false); } @@ -105,7 +106,10 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane try { UserDefinedFileTypesManager.getInstance().setUserDefinedFileTypes(this.fileTypes); } catch (UserDefinedFileTypesManager.UserDefinedFileTypesException ex) { - // RJCTODO + JOptionPane.showMessageDialog(null, + ex.getLocalizedMessage(), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title"), + JOptionPane.ERROR_MESSAGE); } } @@ -120,10 +124,10 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane if (FileTypeIdGlobalSettingsPanel.this.typesList.getSelectedIndex() == -1) { FileTypeIdGlobalSettingsPanel.this.deleteTypeButton.setEnabled(false); } else { - String typeName = FileTypeIdGlobalSettingsPanel.this.typesList.getSelectedValue(); - FileType fileType = FileTypeIdGlobalSettingsPanel.this.fileTypes.get(typeName); + String mimeType = FileTypeIdGlobalSettingsPanel.this.typesList.getSelectedValue(); + FileType fileType = FileTypeIdGlobalSettingsPanel.this.fileTypes.get(mimeType); Signature signature = fileType.getSignature(); - FileTypeIdGlobalSettingsPanel.this.mimeTypeTextField.setText(typeName); + FileTypeIdGlobalSettingsPanel.this.mimeTypeTextField.setText(mimeType); FileType.Signature.Type sigType = fileType.getSignature().getType(); FileTypeIdGlobalSettingsPanel.this.signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); FileTypeIdGlobalSettingsPanel.this.offsetTextField.setText(Long.toString(signature.getOffset())); @@ -139,11 +143,11 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * Sets the list model for the file types list component. */ private void setFileTypesListModel() { - ArrayList typeNames = new ArrayList(this.fileTypes.keySet()); - Collections.sort(typeNames); + ArrayList mimeTypes = new ArrayList(this.fileTypes.keySet()); + Collections.sort(mimeTypes); this.typesListModel.clear(); - for (String typeName : typeNames) { - this.typesListModel.addElement(typeName); + for (String mimeType : mimeTypes) { + this.typesListModel.addElement(mimeType); } } @@ -393,7 +397,16 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * Get the offset. */ - long offset = Long.parseUnsignedLong(this.offsetTextField.getText()); + long offset; + try { + offset = Long.parseUnsignedLong(this.offsetTextField.getText()); + } catch (NumberFormatException ex) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), + JOptionPane.ERROR_MESSAGE); + return; + } /** * Get the interesting files set details. diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index df575fe18d..577e01aca6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -32,6 +32,8 @@ import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; /** * Detects the type of a file based on signature (magic) values. Posts results @@ -45,6 +47,7 @@ public class FileTypeIdIngestModule implements FileIngestModule { private long jobId; private static final HashMap totalsForIngestJobs = new HashMap<>(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); + private final UserDefinedFileTypeIdentifier userDefinedFileTypeIdentifier; private final TikaFileTypeDetector tikaDetector = new TikaFileTypeDetector(); /** @@ -72,6 +75,7 @@ public class FileTypeIdIngestModule implements FileIngestModule { */ FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { this.settings = settings; + this.userDefinedFileTypeIdentifier = new UserDefinedFileTypeIdentifier(); } /** @@ -87,12 +91,12 @@ public class FileTypeIdIngestModule implements FileIngestModule { * @inheritDoc */ @Override - public ProcessResult process(AbstractFile abstractFile) { + public ProcessResult process(AbstractFile file) { /** * Skip unallocated space and unused blocks files. */ - if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) - || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { + if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { return ProcessResult.OK; } @@ -100,7 +104,7 @@ public class FileTypeIdIngestModule implements FileIngestModule { /** * Skip known files if configured to do so. */ - if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) { + if (settings.skipKnownFiles() && (file.getKnown() == FileKnown.KNOWN)) { return ProcessResult.OK; } @@ -108,18 +112,30 @@ public class FileTypeIdIngestModule implements FileIngestModule { * Filter out very small files to minimize false positives. */ // RJCTODO: Make this size a setting - if (abstractFile.getSize() < MIN_FILE_SIZE) { + if (file.getSize() < MIN_FILE_SIZE) { return ProcessResult.OK; } try { long startTime = System.currentTimeMillis(); + FileType fileType = this.userDefinedFileTypeIdentifier.identify(file); + if (null != fileType) { + BlackboardArtifact getInfoArtifact = file.getGenInfoArtifact(); + BlackboardAttribute typeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID(), FileTypeIdModuleFactory.getModuleName(), fileType.getMimeType()); + getInfoArtifact.addAttribute(typeAttr); - // RJCTODO: Add code here to use nay user-defined signstures first; - // if there is a match, post a intersting file hit, if the user has - // elected alerts - tikaDetector.detectAndSave(abstractFile); + if (fileType.alertOnMatch()) { + String moduleName = FileTypeIdModuleFactory.getModuleName(); + BlackboardArtifact artifact = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); + BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), moduleName, fileType.getFilesSetName()); + artifact.addAttribute(setNameAttribute); + BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY.getTypeID(), moduleName, fileType.getMimeType()); + artifact.addAttribute(ruleNameAttribute); + } + } else { + tikaDetector.detectAndSave(file); + } addToTotals(jobId, (System.currentTimeMillis() - startTime)); return ProcessResult.OK; } catch (TskException ex) { From fe64bac2397169ae62043729dbb7e1217f8b3b55 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 19 Dec 2014 17:04:21 -0500 Subject: [PATCH 69/84] Improve user-defined file types UI --- .../sleuthkit/autopsy/casemodule/Case.java | 2 +- .../modules/filetypeid/Bundle.properties | 3 +- .../autopsy/modules/filetypeid/FileType.java | 34 +- .../FileTypeIdGlobalSettingsPanel.form | 120 +++--- .../FileTypeIdGlobalSettingsPanel.java | 403 ++++++++++++------ .../filetypeid/FileTypeIdModuleFactory.java | 13 +- .../FileTypeIdOptionsPanelController.java | 3 +- .../UserDefinedFileTypesManager.java | 14 +- .../autopsy/modules/filetypeid/warning16.png | Bin 0 -> 552 bytes 9 files changed, 382 insertions(+), 210 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/warning16.png diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index cb409ad7c7..69c9034c15 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -167,7 +167,7 @@ public class Case implements SleuthkitCase.ErrorObserver { * * @throws IllegalStateException if there is no case open. */ - public static Case getCurrentCase() { + public static Case getCurrentCase() throws IllegalStateException { if (currentCase != null) { return currentCase; } else { diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index e81c0f9e02..fbf61da330 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -15,7 +15,6 @@ FileTypeIdGlobalSettingsPanel.offsetTextField.text= FileTypeIdGlobalSettingsPanel.offsetLabel.text=Offset FileTypeIdGlobalSettingsPanel.postHitCheckBox.text=Post interesting file hit when found FileTypeIdGlobalSettingsPanel.signatureTextField.text= -FileTypeIdGlobalSettingsPanel.jTextArea1.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. FileTypeIdGlobalSettingsPanel.signatureTypeLabel.text=Signature Type FileTypeIdGlobalSettingsPanel.mimeTypeTextField.text= FileTypeIdGlobalSettingsPanel.signatureLabel.text=Signature @@ -34,3 +33,5 @@ FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Si FileTypeIdGlobalSettingsPanel.filesSetNameLabel.text=Files Set Name FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text= FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title=Save Failed +FileTypeIdGlobalSettingsPanel.hintTextArea.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. +FileTypeIdGlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to file type definitions when ingest is running diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java index 55639114ec..513eff58b3 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -25,36 +25,39 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * Represents a named file type characterized by a file signature. + * Represents a file type characterized by a file signature. + *

    + * Thread-safe. */ final class FileType { private final String mimeType; private final Signature signature; - private final String filesSetName; + private final String interestingFilesSetName; private final boolean alert; /** - * Creates a representation of a named file type characterized by a file + * Creates a representation of a file type characterized by a file * signature. * * @param mimeType The mime type to associate with this file type. * @param signature The signature that characterizes the file type. - * @param filesSetName The interesting files set name - * @param alert A flag indicating whether the user wishes to be alerted when - * a file matching this type is encountered. + * @param filesSetName The name of an interesting files set that includes + * files of this type. + * @param alert Whether the user wishes to be alerted when a file matching + * this type is encountered. */ FileType(String mimeType, final Signature signature, String filesSetName, boolean alert) { this.mimeType = mimeType; this.signature = new Signature(signature.getSignatureBytes(), signature.getOffset(), signature.getType()); - this.filesSetName = filesSetName; + this.interestingFilesSetName = filesSetName; this.alert = alert; } /** * Gets the MIME type associated with this file type. * - * @return The type name. + * @return The MIME type. */ String getMimeType() { return this.mimeType; @@ -63,14 +66,14 @@ final class FileType { /** * Gets the signature associated with this file type. * - * @return The file signature. + * @return The signature. */ Signature getSignature() { return new Signature(this.signature.getSignatureBytes(), this.signature.getOffset(), this.signature.getType()); } /** - * Determines whether or not a given file is an instance of this file type. + * Determines whether or not a file is an instance of this file type. * * @param file The file to test. * @return True or false. @@ -90,17 +93,20 @@ final class FileType { } /** - * Gets the interesting files set name assigned to this file type. + * Gets the name of an interesting files set that includes files of this + * type. * - * @return The files set name, possibly empty. + * @return The interesting files set name, possibly empty. */ String getFilesSetName() { - return this.filesSetName; + return this.interestingFilesSetName; } /** * A file signature consisting of a sequence of bytes at a specific offset * within a file. + *

    + * Thread-safe. */ static final class Signature { @@ -170,7 +176,7 @@ final class FileType { boolean containedIn(final AbstractFile file) { try { byte[] buffer = new byte[this.signatureBytes.length]; - int bytesRead = file.read(buffer, offset, this.signatureBytes.length); + int bytesRead = file.read(buffer, this.offset, this.signatureBytes.length); return ((bytesRead == this.signatureBytes.length) && (Arrays.equals(buffer, this.signatureBytes))); } catch (TskCoreException ex) { Signature.logger.log(Level.WARNING, "Error reading from file with objId = " + file.getId(), ex); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index a4615bbc52..8c04fa7747 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -20,55 +20,64 @@ - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + + - - - - - + + + - - - - - - - + - - - - @@ -77,9 +86,9 @@ - + - + @@ -120,7 +129,9 @@ - + + + @@ -145,7 +156,7 @@ - + @@ -235,6 +246,9 @@ + + + @@ -246,14 +260,14 @@ - + - + @@ -263,7 +277,7 @@ - + @@ -294,5 +308,15 @@ + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 754d52b888..b5ca9abc62 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -18,6 +18,9 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.awt.EventQueue; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; @@ -25,11 +28,14 @@ import java.util.Map; import javax.swing.DefaultComboBoxModel; import javax.swing.DefaultListModel; import javax.swing.JOptionPane; +import javax.swing.event.DocumentEvent; +import javax.swing.event.DocumentListener; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; import javax.xml.bind.DatatypeConverter; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; +import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; @@ -56,8 +62,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * Creates a panel to allow a user to make custom file type definitions. */ FileTypeIdGlobalSettingsPanel() { - this.initComponents(); - this.customizeComponents(); + initComponents(); + customizeComponents(); + addIngestJobEventsListener(); } /** @@ -65,24 +72,113 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * Matisse generated code. */ private void customizeComponents() { - this.typesListModel = new DefaultListModel<>(); - this.typesList.setModel(this.typesListModel); - - DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); - sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); - sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); - this.signatureTypeComboBox.setModel(sigTypeComboBoxModel); - - this.postHitCheckBox.setSelected(false); - this.filesSetNameTextField.setEnabled(false); + setFileTypesListModel(); + setSignatureTypeComboBoxModel(); + clearTypeDetailsComponents(); + addTypeListSelectionListener(); + addTextFieldListeners(); } /** - * @inheritDoc + * Sets the list model for the list of file types. */ - @Override - public void saveSettings() { - this.store(); + private void setFileTypesListModel() { + typesListModel = new DefaultListModel<>(); + typesList.setModel(typesListModel); + } + + /** + * Sets the model for the signature type combo box. + */ + private void setSignatureTypeComboBoxModel() { + DefaultComboBoxModel sigTypeComboBoxModel = new DefaultComboBoxModel<>(); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + sigTypeComboBoxModel.addElement(FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + signatureTypeComboBox.setModel(sigTypeComboBoxModel); + signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + } + + private void addTypeListSelectionListener() { + typesList.addListSelectionListener(new ListSelectionListener() { + @Override + public void valueChanged(ListSelectionEvent e) { + if (e.getValueIsAdjusting() == false) { + if (typesList.getSelectedIndex() == -1) { + clearTypeDetailsComponents(); + } else { + populateTypeDetailsComponents(); + } + } + } + }); + } + + /** + * Adds listeners to the text fields that enable and disable the buttons on + * the panel. + */ + private void addTextFieldListeners() { + DocumentListener listener = new DocumentListener() { + @Override + public void changedUpdate(DocumentEvent e) { + enableButtons(); + } + + @Override + public void removeUpdate(DocumentEvent e) { + enableButtons(); + } + + @Override + public void insertUpdate(DocumentEvent e) { + enableButtons(); + } + }; + + mimeTypeTextField.getDocument().addDocumentListener(listener); + offsetTextField.getDocument().addDocumentListener(listener); + signatureTextField.getDocument().addDocumentListener(listener); + } + + /** + * Add a property change listener that listens to ingest job events to + * disable the buttons on the panel if ingest is running. This is done to + * prevent changes to user-defined types while the type definitions are in + * use. the components in sync with file ingest. + */ + private void addIngestJobEventsListener() { + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + enableButtons(); + } + }); + } + }); + } + + /** + * Enables or disables the panel buttons based on the state of the panel and + * the application. + */ + private void enableButtons() { + boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning(); + newTypeButton.setEnabled(!ingestIsRunning); + + boolean fileTypeIsSelected = typesList.getSelectedIndex() != -1; + deleteTypeButton.setEnabled(!ingestIsRunning && fileTypeIsSelected); + + boolean requiredFieldsPopulated + = !mimeTypeTextField.getText().isEmpty() + && !offsetTextField.getText().isEmpty() + && !signatureTextField.getText().isEmpty() + && postHitCheckBox.isSelected() ? !filesSetNameTextField.getText().isEmpty() : true; + saveTypeButton.setEnabled(!ingestIsRunning && fileTypeIsSelected && requiredFieldsPopulated); + + ingestRunningWarningLabel.setVisible(ingestIsRunning); } /** @@ -90,12 +186,59 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane */ @Override public void load() { - this.fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); - this.setFileTypesListModel(); - this.typesList.addListSelectionListener(new TypesListSelectionListener()); - if (!this.typesListModel.isEmpty()) { - this.typesList.setSelectedIndex(0); + fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); + updateFileTypesListModel(); + if (!typesListModel.isEmpty()) { + typesList.setSelectedIndex(0); } + enableButtons(); + } + + /** + * Sets the list model for the file types list component. + */ + private void updateFileTypesListModel() { + ArrayList mimeTypes = new ArrayList<>(fileTypes.keySet()); + Collections.sort(mimeTypes); + typesListModel.clear(); + for (String mimeType : mimeTypes) { + typesListModel.addElement(mimeType); + } + } + + /** + * Sets all of the components in the individual type details portion of the + * panel based on the current selection in the file types list. + */ + private void populateTypeDetailsComponents() { + String mimeType = typesList.getSelectedValue(); + FileType fileType = fileTypes.get(mimeType); + if (null != fileType) { + Signature signature = fileType.getSignature(); + mimeTypeTextField.setText(mimeType); + FileType.Signature.Type sigType = fileType.getSignature().getType(); + signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + offsetTextField.setText(Long.toString(signature.getOffset())); + postHitCheckBox.setSelected(fileType.alertOnMatch()); + filesSetNameTextField.setText(fileType.getFilesSetName()); + } + enableButtons(); + } + + /** + * Clears all of the components in the individual type details portion of + * the panel. + */ + private void clearTypeDetailsComponents() { + typesList.setSelectedIndex(-1); + mimeTypeTextField.setText(""); //NON-NLS + signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + signatureTextField.setText(""); //NON-NLS + offsetTextField.setText(""); //NON-NLS + postHitCheckBox.setSelected(false); + filesSetNameTextField.setText(""); //NON-NLS + filesSetNameTextField.setEnabled(false); + enableButtons(); } /** @@ -104,7 +247,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane @Override public void store() { try { - UserDefinedFileTypesManager.getInstance().setUserDefinedFileTypes(this.fileTypes); + UserDefinedFileTypesManager.getInstance().setUserDefinedFileTypes(fileTypes); } catch (UserDefinedFileTypesManager.UserDefinedFileTypesException ex) { JOptionPane.showMessageDialog(null, ex.getLocalizedMessage(), @@ -114,57 +257,13 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } /** - * Selection listener for the file types list component. + * @inheritDoc */ - private class TypesListSelectionListener implements ListSelectionListener { - - @Override - public void valueChanged(ListSelectionEvent e) { - if (e.getValueIsAdjusting() == false) { - if (FileTypeIdGlobalSettingsPanel.this.typesList.getSelectedIndex() == -1) { - FileTypeIdGlobalSettingsPanel.this.deleteTypeButton.setEnabled(false); - } else { - String mimeType = FileTypeIdGlobalSettingsPanel.this.typesList.getSelectedValue(); - FileType fileType = FileTypeIdGlobalSettingsPanel.this.fileTypes.get(mimeType); - Signature signature = fileType.getSignature(); - FileTypeIdGlobalSettingsPanel.this.mimeTypeTextField.setText(mimeType); - FileType.Signature.Type sigType = fileType.getSignature().getType(); - FileTypeIdGlobalSettingsPanel.this.signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); - FileTypeIdGlobalSettingsPanel.this.offsetTextField.setText(Long.toString(signature.getOffset())); - FileTypeIdGlobalSettingsPanel.this.postHitCheckBox.setSelected(fileType.alertOnMatch()); - FileTypeIdGlobalSettingsPanel.this.filesSetNameTextField.setText(fileType.getFilesSetName()); - FileTypeIdGlobalSettingsPanel.this.deleteTypeButton.setEnabled(true); - } - } - } + @Override + public void saveSettings() { + store(); } - - /** - * Sets the list model for the file types list component. - */ - private void setFileTypesListModel() { - ArrayList mimeTypes = new ArrayList(this.fileTypes.keySet()); - Collections.sort(mimeTypes); - this.typesListModel.clear(); - for (String mimeType : mimeTypes) { - this.typesListModel.addElement(mimeType); - } - } - - /** - * Clears all of the components in the individual type details portion of - * the panel. - */ - private void clearTypeDetailsComponents() { - this.typesList.setSelectedIndex(-1); - this.mimeTypeTextField.setText(""); - this.signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); - this.signatureTextField.setText(""); //NON-NLS - this.offsetTextField.setText(""); //NON-NLS - this.postHitCheckBox.setSelected(false); - this.filesSetNameTextField.setText(""); - } - + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -176,7 +275,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane typesScrollPane = new javax.swing.JScrollPane(); typesList = new javax.swing.JList(); - jSeparator1 = new javax.swing.JSeparator(); + separator = new javax.swing.JSeparator(); mimeTypeLabel = new javax.swing.JLabel(); mimeTypeTextField = new javax.swing.JTextField(); signatureTypeLabel = new javax.swing.JLabel(); @@ -189,15 +288,16 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane hexPrefixLabel = new javax.swing.JLabel(); signatureTypeComboBox = new javax.swing.JComboBox(); signatureLabel = new javax.swing.JLabel(); - jScrollPane2 = new javax.swing.JScrollPane(); - jTextArea1 = new javax.swing.JTextArea(); + hintScrollPane = new javax.swing.JScrollPane(); + hintTextArea = new javax.swing.JTextArea(); postHitCheckBox = new javax.swing.JCheckBox(); filesSetNameLabel = new javax.swing.JLabel(); filesSetNameTextField = new javax.swing.JTextField(); + ingestRunningWarningLabel = new javax.swing.JLabel(); typesScrollPane.setViewportView(typesList); - jSeparator1.setOrientation(javax.swing.SwingConstants.VERTICAL); + separator.setOrientation(javax.swing.SwingConstants.VERTICAL); org.openide.awt.Mnemonics.setLocalizedText(mimeTypeLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.mimeTypeLabel.text")); // NOI18N @@ -234,16 +334,22 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane org.openide.awt.Mnemonics.setLocalizedText(hexPrefixLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text")); // NOI18N + signatureTypeComboBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + signatureTypeComboBoxActionPerformed(evt); + } + }); + org.openide.awt.Mnemonics.setLocalizedText(signatureLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.signatureLabel.text")); // NOI18N - jTextArea1.setEditable(false); - jTextArea1.setColumns(20); - jTextArea1.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N - jTextArea1.setLineWrap(true); - jTextArea1.setRows(5); - jTextArea1.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.jTextArea1.text")); // NOI18N - jTextArea1.setWrapStyleWord(true); - jScrollPane2.setViewportView(jTextArea1); + hintTextArea.setEditable(false); + hintTextArea.setColumns(20); + hintTextArea.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N + hintTextArea.setLineWrap(true); + hintTextArea.setRows(5); + hintTextArea.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.hintTextArea.text")); // NOI18N + hintTextArea.setWrapStyleWord(true); + hintScrollPane.setViewportView(hintTextArea); org.openide.awt.Mnemonics.setLocalizedText(postHitCheckBox, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.postHitCheckBox.text")); // NOI18N postHitCheckBox.addActionListener(new java.awt.event.ActionListener() { @@ -256,6 +362,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane filesSetNameTextField.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text")); // NOI18N + ingestRunningWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/filetypeid/warning16.png"))); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(ingestRunningWarningLabel, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.ingestRunningWarningLabel.text")); // NOI18N + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -264,53 +373,58 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addGap(26, 26, 26) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() - .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)) - .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() - .addGap(0, 0, Short.MAX_VALUE) - .addComponent(newTypeButton) + .addComponent(ingestRunningWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGap(30, 30, 30)) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addGap(0, 0, Short.MAX_VALUE) + .addComponent(newTypeButton) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(deleteTypeButton) + .addGap(9, 9, 9))) + .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(deleteTypeButton) - .addGap(9, 9, 9))) - .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) - .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() - .addComponent(signatureTypeLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(signatureTypeComboBox, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) - .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(offsetLabel) - .addComponent(filesSetNameLabel) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addGroup(layout.createSequentialGroup() - .addComponent(signatureLabel) - .addGap(18, 18, 18) - .addComponent(hexPrefixLabel))) - .addGap(5, 5, 5) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 84, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() - .addGap(2, 2, 2) - .addComponent(mimeTypeLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 181, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 179, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(postHitCheckBox) - .addComponent(saveTypeButton)) - .addContainerGap(29, Short.MAX_VALUE)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(offsetLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 71, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(filesSetNameLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 179, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) + .addComponent(signatureLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(signatureTypeLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() + .addGap(2, 2, 2) + .addComponent(mimeTypeLabel))) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 176, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 176, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addComponent(hexPrefixLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 160, javax.swing.GroupLayout.PREFERRED_SIZE))))) + .addComponent(hintScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(postHitCheckBox) + .addComponent(saveTypeButton)) + .addGap(29, 29, 29)))) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(16, 16, 16) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) - .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 281, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 281, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(layout.createSequentialGroup() - .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(hintScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(mimeTypeLabel) @@ -342,19 +456,21 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(deleteTypeButton) .addComponent(newTypeButton)))) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(ingestRunningWarningLabel) + .addContainerGap(22, Short.MAX_VALUE)) ); }// //GEN-END:initComponents private void newTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_newTypeButtonActionPerformed - this.clearTypeDetailsComponents(); + clearTypeDetailsComponents(); }//GEN-LAST:event_newTypeButtonActionPerformed private void deleteTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteTypeButtonActionPerformed - String typeName = this.typesList.getSelectedValue(); - this.fileTypes.remove(typeName); - this.clearTypeDetailsComponents(); - this.typesList.setSelectedIndex(-1); + String typeName = typesList.getSelectedValue(); + fileTypes.remove(typeName); + clearTypeDetailsComponents(); + typesList.setSelectedIndex(-1); }//GEN-LAST:event_deleteTypeButtonActionPerformed private void saveTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_saveTypeButtonActionPerformed @@ -362,7 +478,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * Get the file type name. */ - String typeName = this.mimeTypeTextField.getText(); + String typeName = mimeTypeTextField.getText(); if (typeName.isEmpty()) { JOptionPane.showMessageDialog(null, NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.message"), @@ -374,12 +490,12 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * Get the signature type. */ - FileType.Signature.Type sigType = this.signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; + FileType.Signature.Type sigType = signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; /** * Get the signature bytes. */ - String sigString = this.signatureTextField.getText(); + String sigString = signatureTextField.getText(); if (sigString.isEmpty()) { JOptionPane.showMessageDialog(null, NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message"), @@ -399,7 +515,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane */ long offset; try { - offset = Long.parseUnsignedLong(this.offsetTextField.getText()); + offset = Long.parseUnsignedLong(offsetTextField.getText()); } catch (NumberFormatException ex) { JOptionPane.showMessageDialog(null, NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), @@ -411,8 +527,8 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * Get the interesting files set details. */ - String filesSetName = this.filesSetNameTextField.getText(); - if (this.postHitCheckBox.isSelected() && filesSetName.isEmpty()) { + String filesSetName = filesSetNameTextField.getText(); + if (postHitCheckBox.isSelected() && filesSetName.isEmpty()) { JOptionPane.showMessageDialog(null, NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), @@ -423,10 +539,10 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * Put it all together and reset the file types list component. */ FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); - FileType fileType = new FileType(typeName, signature, filesSetName, this.postHitCheckBox.isSelected()); - this.fileTypes.put(typeName, fileType); - this.setFileTypesListModel(); - this.typesList.setSelectedValue(fileType.getMimeType(), true); + FileType fileType = new FileType(typeName, signature, filesSetName, postHitCheckBox.isSelected()); + fileTypes.put(typeName, fileType); + updateFileTypesListModel(); + typesList.setSelectedValue(fileType.getMimeType(), true); } catch (NumberFormatException ex) { JOptionPane.showMessageDialog(null, @@ -442,17 +558,21 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane }//GEN-LAST:event_saveTypeButtonActionPerformed private void postHitCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_postHitCheckBoxActionPerformed - this.filesSetNameTextField.setEnabled(this.postHitCheckBox.isSelected()); + filesSetNameTextField.setEnabled(postHitCheckBox.isSelected()); }//GEN-LAST:event_postHitCheckBoxActionPerformed + private void signatureTypeComboBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_signatureTypeComboBoxActionPerformed + hexPrefixLabel.setVisible(signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + }//GEN-LAST:event_signatureTypeComboBoxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton deleteTypeButton; private javax.swing.JLabel filesSetNameLabel; private javax.swing.JTextField filesSetNameTextField; private javax.swing.JLabel hexPrefixLabel; - private javax.swing.JScrollPane jScrollPane2; - private javax.swing.JSeparator jSeparator1; - private javax.swing.JTextArea jTextArea1; + private javax.swing.JScrollPane hintScrollPane; + private javax.swing.JTextArea hintTextArea; + private javax.swing.JLabel ingestRunningWarningLabel; private javax.swing.JLabel mimeTypeLabel; private javax.swing.JTextField mimeTypeTextField; private javax.swing.JButton newTypeButton; @@ -460,6 +580,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private javax.swing.JTextField offsetTextField; private javax.swing.JCheckBox postHitCheckBox; private javax.swing.JButton saveTypeButton; + private javax.swing.JSeparator separator; private javax.swing.JLabel signatureLabel; private javax.swing.JTextField signatureTextField; private javax.swing.JComboBox signatureTypeComboBox; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java index dbd54351cc..ee6f427dde 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java @@ -34,6 +34,12 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; @ServiceProvider(service = IngestModuleFactory.class) public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { + /** + * Make this a singleton since it is a ingest job event listener that never + * de-registers. + */ + FileTypeIdGlobalSettingsPanel globalSettingsPanel; + /** * @inheritDoc */ @@ -82,9 +88,12 @@ public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { */ @Override public IngestModuleGlobalSettingsPanel getGlobalSettingsPanel() { - return new FileTypeIdGlobalSettingsPanel(); + if (null == globalSettingsPanel) { + globalSettingsPanel = new FileTypeIdGlobalSettingsPanel(); + } + return globalSettingsPanel; } - + /** * @inheritDoc */ diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java index bbf16073de..8f9aa7b500 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java @@ -8,14 +8,13 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import javax.swing.JComponent; -import javax.swing.SwingUtilities; import org.netbeans.spi.options.OptionsPanelController; import org.openide.util.HelpCtx; import org.openide.util.Lookup; @OptionsPanelController.TopLevelRegistration( categoryName = "#OptionsCategory_Name_FileTypeId", - iconBase = "org/sleuthkit/autopsy/corecomponents/display-options.png", + iconBase = "org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png", keywords = "#OptionsCategory_Keywords_FileTypeId", keywordsCategory = "FileTypeId" ) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 65a7a447a2..4c1325192c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -45,6 +45,18 @@ import org.xml.sax.SAXException; /** * Manages user-defined file types characterized by MIME type, signature, and * optional membership in an interesting files set. + *

    + * Note that this class exposes a very simple get/set API that operates on the + * user-defined file types as a complete set - there is no concept of adding, + * editing or deleting file types singly. This works because this class is not + * exposed outside of this ingest module package and is ONLY used in a very + * specific paradigm where there is a single modal writer of file types in the + * form of a global settings panel that disables itself when ingest is running + * so that multiple readers in the form of file ingest modules get a consistent + * set of file type definitions. Moreover, there is no enforcement of compliance + * with this paradigm by this class - it merely participates in the scheme. + *

    + * Thread-safe. */ final class UserDefinedFileTypesManager { @@ -87,7 +99,7 @@ final class UserDefinedFileTypesManager { * thread-safety. */ private final Map fileTypes = new HashMap<>(); - + /** * Gets the manager of user-defined file types characterized by MIME type, * signature, and optional membership in an interesting files set. diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/warning16.png b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/warning16.png new file mode 100644 index 0000000000000000000000000000000000000000..f5ba881738ae3072e476f3ddbd7dd34d642f06d6 GIT binary patch literal 552 zcmV+@0@wYCP)A&a_P7S}o zG!s6}jEu~GKp4p&41mvo3NbA|1~KhGFq>2ZZB(et$g5@cY9-1{tFW22MeRe4v~wF$QERIaM<- z{{O*nd`23>@mYBc|Np&XP;x8-%MmglnP1j4N=(g;;pf|paKWFi*D(ky8Zq$8n1Dhw z4W9v-%1)IGf4<*f`2Xh#iNdNA(_lT4EX==BZIP28N-#eIhh~>reEIz qPU=w%q=Bs Date: Fri, 19 Dec 2014 17:04:37 -0500 Subject: [PATCH 70/84] Improve user-defined file types UI --- .../user-defined-file-types-settings.png | Bin 0 -> 1652 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png new file mode 100644 index 0000000000000000000000000000000000000000..4dc0c47b17696e6eca344dfbb1fcb2fa7a588661 GIT binary patch literal 1652 zcmV-)28;QLP)!3MP&iN;Q8y_;930mYIZ#D45)wJ#K(R=2 zpb>&Zju4cH5)@G(6-1?o+}cB{G^8Bp0X5--gdL=S>_#ajcGB3JZTxR{-+TLJhQq#f zW4F6r(gXaY(c4+=%zo$l{r=|nd+_Bhr5?#< zv)xjz`x0@G+B(hFo}MR- zF;5l>h1x=~_(m?5`@zwpNB`Aojk6JeHKsnSMQe?<7RPZALLd^?wi>SMlF4M$_U+q8 z2KVjz-Opcm;j{O}fa^F&*ToovF@`vfSy@@Z7=vh?9)v-F5 z$&>$V3qXSbzavf7SXNh-vBqGuCJaN2F)d|qU6)iUMHoc@^!4?9Wq5e_*Ncmb-#T;V z%)d96z*>Tu=GAMO-~Iz`u3AV~u?ZWkhSr+Za+zwiiqRS&1pWQ}gZuaI|Ha72NPcq( zj3KsAj69a`(JW)E!4ZO7Hq$gEEnsb3f?BOcu~=kgW`-aL5W?Y;`TR2n4jhcfxXjyvqGVd?1Qv2La;9+jAXkLCt5epXwkPdYD zknez%2*R3WsJqM#wL+{77LZcX)6+vPmqRH{m(;v&+|5aScY0_A7veBdFJ-$f?<3AR2E^Vhfj%xkZ` z&dkgV*-VD7J^eI?Mn_4dQuR#PVB(MTfa?nSx?H*yL^XpphH52Z_Qo<=-^C)ZI0z|m z)ODOx4#(?cu_NHe$HvzE!QUn(DBZckv17-OQhq3lKGFk12&zFqxl%<4fzcXAo{@!7FdwV&5{ye{W@kM@r?i^!dW8`uLCxhtsD|=bvYGc9!3~ z^b*^)ZCl%8qXf=+4}4HiYH^LI7S%hXM2?2`n~k?GUE=cP%N#s-klxS8p6BI^G0Iwlr&L=KOiWC0;lc$51_s!v({z)#A)|zJ2It#^^_lu?W##umAOtXL#a?0Y3BT{dMQ5&r>cti$kf)k{|Z0000 Date: Sun, 21 Dec 2014 21:31:10 -0500 Subject: [PATCH 71/84] Complete user-defined file types feature --- .../sleuthkit/autopsy/coreutils/XMLUtil.java | 10 +-- .../autopsy/modules/filetypeid/FileType.java | 39 +++++---- .../FileTypeIdGlobalSettingsPanel.java | 81 +++++++++++-------- .../FileTypeIdIngestJobSettingsPanel.java | 10 ++- .../filetypeid/FileTypeIdIngestModule.java | 10 ++- .../filetypeid/FileTypeIdModuleFactory.java | 5 +- .../filetypeid/FileTypeIdModuleSettings.java | 2 +- .../UserDefinedFileTypesManager.java | 4 +- 8 files changed, 94 insertions(+), 67 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java index ef360762d9..cc055638d7 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java @@ -167,7 +167,7 @@ public class XMLUtil { * IngestModuleLoader. * */ - // RJCTODO: Deprecate. + // TODO: Deprecate. public static boolean xmlIsValid(DOMSource xmlfile, Class clazz, String schemaFile) { try { PlatformUtil.extractResourceToUserConfigDir(clazz, schemaFile, false); @@ -205,7 +205,7 @@ public class XMLUtil { * IngestModuleLoader. * */ - // RJCTODO: Deprecate. + // TODO: Deprecate. public static boolean xmlIsValid(Document doc, Class clazz, String type) { DOMSource dms = new DOMSource(doc); return xmlIsValid(dms, clazz, type); @@ -217,7 +217,7 @@ public class XMLUtil { * @param clazz the class this method is invoked from * @param xmlPath the full path to the file to load */ - // RJCTODO: Deprecate. + // TODO: Deprecate. public static Document loadDoc(Class clazz, String xmlPath) { DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); Document ret = null; @@ -242,7 +242,7 @@ public class XMLUtil { * @param xmlPath the full path to the file to load * @param xsdPath the full path to the file to validate against */ - // RJCTODO: Deprecate + // TODO: Deprecate public static Document loadDoc(Class clazz, String xmlPath, String xsdPath) { Document ret = loadDoc(clazz, xmlPath); if (!XMLUtil.xmlIsValid(ret, clazz, xsdPath)) { @@ -259,7 +259,7 @@ public class XMLUtil { * @param encoding to encoding, such as "UTF-8", to encode the file with * @param doc the document to save */ - // RJCTODO: Deprecate. + // TODO: Deprecate. public static boolean saveDoc(Class clazz, String xmlPath, String encoding, final Document doc) { TransformerFactory xf = TransformerFactory.newInstance(); xf.setAttribute("indent-number", 1); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java index 513eff58b3..e3903ffc30 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -27,9 +27,9 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Represents a file type characterized by a file signature. *

    - * Thread-safe. + * Thread-safe (immutable). */ -final class FileType { +class FileType { private final String mimeType; private final Signature signature; @@ -41,9 +41,9 @@ final class FileType { * signature. * * @param mimeType The mime type to associate with this file type. - * @param signature The signature that characterizes the file type. + * @param signature The signature that characterizes this file type. * @param filesSetName The name of an interesting files set that includes - * files of this type. + * files of this type, may be the empty string. * @param alert Whether the user wishes to be alerted when a file matching * this type is encountered. */ @@ -60,7 +60,7 @@ final class FileType { * @return The MIME type. */ String getMimeType() { - return this.mimeType; + return mimeType; } /** @@ -69,7 +69,7 @@ final class FileType { * @return The signature. */ Signature getSignature() { - return new Signature(this.signature.getSignatureBytes(), this.signature.getOffset(), this.signature.getType()); + return new Signature(signature.getSignatureBytes(), signature.getOffset(), signature.getType()); } /** @@ -79,7 +79,7 @@ final class FileType { * @return True or false. */ boolean matches(final AbstractFile file) { - return this.signature.containedIn(file); + return signature.containedIn(file); } /** @@ -89,7 +89,7 @@ final class FileType { * @return True or false. */ boolean alertOnMatch() { - return this.alert; + return alert; } /** @@ -99,16 +99,16 @@ final class FileType { * @return The interesting files set name, possibly empty. */ String getFilesSetName() { - return this.interestingFilesSetName; + return interestingFilesSetName; } /** * A file signature consisting of a sequence of bytes at a specific offset * within a file. *

    - * Thread-safe. + * Thread-safe (immutable). */ - static final class Signature { + static class Signature { private static final Logger logger = Logger.getLogger(Signature.class.getName()); @@ -145,7 +145,7 @@ final class FileType { * @return The byte sequence as an array of bytes. */ byte[] getSignatureBytes() { - return Arrays.copyOf(this.signatureBytes, this.signatureBytes.length); + return Arrays.copyOf(signatureBytes, signatureBytes.length); } /** @@ -154,7 +154,7 @@ final class FileType { * @return The offset. */ long getOffset() { - return this.offset; + return offset; } /** @@ -163,7 +163,7 @@ final class FileType { * @return The signature type. */ Type getType() { - return this.type; + return type; } /** @@ -175,10 +175,15 @@ final class FileType { */ boolean containedIn(final AbstractFile file) { try { - byte[] buffer = new byte[this.signatureBytes.length]; - int bytesRead = file.read(buffer, this.offset, this.signatureBytes.length); - return ((bytesRead == this.signatureBytes.length) && (Arrays.equals(buffer, this.signatureBytes))); + byte[] buffer = new byte[signatureBytes.length]; + int bytesRead = file.read(buffer, offset, signatureBytes.length); + return ((bytesRead == signatureBytes.length) && (Arrays.equals(buffer, signatureBytes))); } catch (TskCoreException ex) { + /** + * This exception is caught rather than propagated because files + * in images are not always consistent with their file system + * meta data making for read errors. + */ Signature.logger.log(Level.WARNING, "Error reading from file with objId = " + file.getId(), ex); return false; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index b5ca9abc62..4984384296 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -33,6 +33,7 @@ import javax.swing.event.DocumentListener; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; import javax.xml.bind.DatatypeConverter; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestManager; @@ -58,6 +59,14 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private DefaultListModel typesListModel; private Map fileTypes; + /** + * This panel implements a property change listener that listens to ingest + * job events to disable the buttons on the panel if ingest is running. This + * is done to prevent changes to user-defined types while the type + * definitions are in use. + */ + IngestJobEventPropertyChangeListener ingestJobEventsListener; + /** * Creates a panel to allow a user to make custom file type definitions. */ @@ -110,9 +119,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } } } - }); + }); } - + /** * Adds listeners to the text fields that enable and disable the buttons on * the panel. @@ -144,20 +153,27 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * Add a property change listener that listens to ingest job events to * disable the buttons on the panel if ingest is running. This is done to * prevent changes to user-defined types while the type definitions are in - * use. the components in sync with file ingest. + * use. */ private void addIngestJobEventsListener() { - IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { - @Override - public void propertyChange(PropertyChangeEvent evt) { - EventQueue.invokeLater(new Runnable() { - @Override - public void run() { - enableButtons(); - } - }); - } - }); + ingestJobEventsListener = new IngestJobEventPropertyChangeListener(); + IngestManager.getInstance().addIngestJobEventListener(ingestJobEventsListener); + } + + /** + * A property change listener that listens to ingest job events. + */ + private class IngestJobEventPropertyChangeListener implements PropertyChangeListener { + + @Override + public void propertyChange(PropertyChangeEvent evt) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + enableButtons(); + } + }); + } } /** @@ -177,7 +193,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane && !signatureTextField.getText().isEmpty() && postHitCheckBox.isSelected() ? !filesSetNameTextField.getText().isEmpty() : true; saveTypeButton.setEnabled(!ingestIsRunning && fileTypeIsSelected && requiredFieldsPopulated); - + ingestRunningWarningLabel.setVisible(ingestIsRunning); } @@ -207,21 +223,21 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane } /** - * Sets all of the components in the individual type details portion of the + * Populates all of the components in the file type details portion of the * panel based on the current selection in the file types list. */ private void populateTypeDetailsComponents() { String mimeType = typesList.getSelectedValue(); FileType fileType = fileTypes.get(mimeType); if (null != fileType) { - Signature signature = fileType.getSignature(); mimeTypeTextField.setText(mimeType); - FileType.Signature.Type sigType = fileType.getSignature().getType(); + Signature signature = fileType.getSignature(); + FileType.Signature.Type sigType = signature.getType(); signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); offsetTextField.setText(Long.toString(signature.getOffset())); postHitCheckBox.setSelected(fileType.alertOnMatch()); filesSetNameTextField.setText(fileType.getFilesSetName()); - } + } enableButtons(); } @@ -233,6 +249,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane typesList.setSelectedIndex(-1); mimeTypeTextField.setText(""); //NON-NLS signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); + hexPrefixLabel.setVisible(true); signatureTextField.setText(""); //NON-NLS offsetTextField.setText(""); //NON-NLS postHitCheckBox.setSelected(false); @@ -263,7 +280,17 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane public void saveSettings() { store(); } - + + /** + * @inheritDoc + */ + @Override + @SuppressWarnings("FinalizeDeclaration") + protected void finalize() throws Throwable { + IngestManager.getInstance().removeIngestJobEventListener(ingestJobEventsListener); + super.finalize(); + } + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -470,13 +497,12 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane String typeName = typesList.getSelectedValue(); fileTypes.remove(typeName); clearTypeDetailsComponents(); - typesList.setSelectedIndex(-1); }//GEN-LAST:event_deleteTypeButtonActionPerformed private void saveTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_saveTypeButtonActionPerformed try { /** - * Get the file type name. + * Get the MIME type. */ String typeName = mimeTypeTextField.getText(); if (typeName.isEmpty()) { @@ -513,16 +539,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * Get the offset. */ - long offset; - try { - offset = Long.parseUnsignedLong(offsetTextField.getText()); - } catch (NumberFormatException ex) { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), - JOptionPane.ERROR_MESSAGE); - return; - } + long offset = Long.parseUnsignedLong(offsetTextField.getText()); /** * Get the interesting files set details. diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java index f31380379d..ff65c1ac63 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java @@ -25,12 +25,10 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; * UI component used to set ingest job options for file type identifier ingest * modules. */ -final class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPanel { +class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPanel { private final FileTypeIdModuleSettings settings; - // NOTE: This was declared public, but was inaccessible because the class is - // not public FileTypeIdIngestJobSettingsPanel(FileTypeIdModuleSettings settings) { this.settings = settings; initComponents(); @@ -44,7 +42,11 @@ final class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettin public IngestModuleIngestJobSettings getSettings() { return settings; } - + + /** + * Does child component initialization in addition to that done by the + * Matisse generated code. + */ private void customizeComponents() { skipKnownCheckBox.setSelected(settings.skipKnownFiles()); } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 577e01aca6..348bb13e9c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -39,6 +39,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute; * Detects the type of a file based on signature (magic) values. Posts results * to the blackboard. */ +// TODO: This class does not need to be public. public class FileTypeIdIngestModule implements FileIngestModule { private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); @@ -120,15 +121,20 @@ public class FileTypeIdIngestModule implements FileIngestModule { long startTime = System.currentTimeMillis(); FileType fileType = this.userDefinedFileTypeIdentifier.identify(file); if (null != fileType) { + String moduleName = FileTypeIdModuleFactory.getModuleName(); BlackboardArtifact getInfoArtifact = file.getGenInfoArtifact(); - BlackboardAttribute typeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID(), FileTypeIdModuleFactory.getModuleName(), fileType.getMimeType()); + BlackboardAttribute typeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID(), moduleName, fileType.getMimeType()); getInfoArtifact.addAttribute(typeAttr); if (fileType.alertOnMatch()) { - String moduleName = FileTypeIdModuleFactory.getModuleName(); BlackboardArtifact artifact = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), moduleName, fileType.getFilesSetName()); artifact.addAttribute(setNameAttribute); + + /** + * Use the MIME type as the category, i.e., the rule that + * determined this file belongs to the interesting files set. + */ BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY.getTypeID(), moduleName, fileType.getMimeType()); artifact.addAttribute(ruleNameAttribute); } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java index ee6f427dde..f831ed15e2 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java @@ -31,13 +31,10 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; /** * A factory that creates file ingest modules that determine the types of files. */ +// TODO: This class does not need to be public. @ServiceProvider(service = IngestModuleFactory.class) public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { - /** - * Make this a singleton since it is a ingest job event listener that never - * de-registers. - */ FileTypeIdGlobalSettingsPanel globalSettingsPanel; /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java index 31e077a875..ddf5371045 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java @@ -23,7 +23,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; /** * Ingest job options for the file type identifier ingest module instances. */ -// @Deprecated This class is not for public use +// TODO: This class does not need to be public. public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { private static final long serialVersionUID = 1L; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 4c1325192c..9f093b5570 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -268,10 +268,10 @@ final class UserDefinedFileTypesManager { } catch (ParserConfigurationException | FileNotFoundException | UnsupportedEncodingException | TransformerException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); - throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message + throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); } catch (IOException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); - throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); // RJCTODO: Create a bundled message + throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); } /** From a9ea3d280e43ed9baee5ec5bf5ecfdffe93f2a59 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sun, 21 Dec 2014 23:48:43 -0500 Subject: [PATCH 72/84] Bug fixes for user-defined file types --- .../FileTypeIdGlobalSettingsPanel.java | 9 +++++--- .../filetypeid/FileTypeIdIngestModule.java | 6 ++--- .../filetypeid/FileTypeIdModuleFactory.java | 1 + .../filetypeid/UserDefinedFileTypes.xsd | 16 +++++++------- .../UserDefinedFileTypesManager.java | 22 +++++++++---------- 5 files changed, 29 insertions(+), 25 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 4984384296..83bc32d167 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -147,6 +147,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane mimeTypeTextField.getDocument().addDocumentListener(listener); offsetTextField.getDocument().addDocumentListener(listener); signatureTextField.getDocument().addDocumentListener(listener); + filesSetNameTextField.getDocument().addDocumentListener(listener); } /** @@ -186,13 +187,13 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane boolean fileTypeIsSelected = typesList.getSelectedIndex() != -1; deleteTypeButton.setEnabled(!ingestIsRunning && fileTypeIsSelected); - + boolean requiredFieldsPopulated = !mimeTypeTextField.getText().isEmpty() && !offsetTextField.getText().isEmpty() && !signatureTextField.getText().isEmpty() - && postHitCheckBox.isSelected() ? !filesSetNameTextField.getText().isEmpty() : true; - saveTypeButton.setEnabled(!ingestIsRunning && fileTypeIsSelected && requiredFieldsPopulated); + && (postHitCheckBox.isSelected() ? !filesSetNameTextField.getText().isEmpty() : true); + saveTypeButton.setEnabled(!ingestIsRunning && requiredFieldsPopulated); ingestRunningWarningLabel.setVisible(ingestIsRunning); } @@ -234,6 +235,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane Signature signature = fileType.getSignature(); FileType.Signature.Type sigType = signature.getType(); signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); + this.signatureTextField.setText(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); offsetTextField.setText(Long.toString(signature.getOffset())); postHitCheckBox.setSelected(fileType.alertOnMatch()); filesSetNameTextField.setText(fileType.getFilesSetName()); @@ -576,6 +578,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private void postHitCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_postHitCheckBoxActionPerformed filesSetNameTextField.setEnabled(postHitCheckBox.isSelected()); + enableButtons(); }//GEN-LAST:event_postHitCheckBoxActionPerformed private void signatureTypeComboBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_signatureTypeComboBoxActionPerformed diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 348bb13e9c..6d6a1796e5 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -113,9 +113,9 @@ public class FileTypeIdIngestModule implements FileIngestModule { * Filter out very small files to minimize false positives. */ // RJCTODO: Make this size a setting - if (file.getSize() < MIN_FILE_SIZE) { - return ProcessResult.OK; - } +// if (file.getSize() < MIN_FILE_SIZE) { +// return ProcessResult.OK; +// } try { long startTime = System.currentTimeMillis(); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java index f831ed15e2..38823ca03d 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java @@ -88,6 +88,7 @@ public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { if (null == globalSettingsPanel) { globalSettingsPanel = new FileTypeIdGlobalSettingsPanel(); } + globalSettingsPanel.load(); return globalSettingsPanel; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd index 47f658599a..727d86bb97 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd @@ -24,22 +24,22 @@ - - - + + + - + - - - + + + - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 9f093b5570..74762cbbfb 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -63,14 +63,14 @@ final class UserDefinedFileTypesManager { private static final Logger logger = Logger.getLogger(UserDefinedFileTypesManager.class.getName()); private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; //NON-NLS private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; //NON-NLS - private static final String FILE_TYPES_TAG_NAME = "filetypes"; //NON-NLS - private static final String FILE_TYPE_TAG_NAME = "filetype"; //NON-NLS - private static final String MIME_TYPE_TAG_NAME = "mimetype"; //NON-NLS - private static final String SIGNATURE_TAG_NAME = "signature"; //NON-NLS + private static final String FILE_TYPES_TAG_NAME = "FileTypes"; //NON-NLS + private static final String FILE_TYPE_TAG_NAME = "FileType"; //NON-NLS + private static final String MIME_TYPE_TAG_NAME = "MimeType"; //NON-NLS + private static final String SIGNATURE_TAG_NAME = "Signature"; //NON-NLS private static final String SIGNATURE_TYPE_ATTRIBUTE = "type"; //NON-NLS - private static final String BYTES_TAG_NAME = "bytes"; //NON-NLS - private static final String OFFSET_TAG_NAME = "offset"; //NON-NLS - private static final String INTERESTING_FILES_SET_TAG_NAME = "filesset"; //NON-NLS + private static final String BYTES_TAG_NAME = "Bytes"; //NON-NLS + private static final String OFFSET_TAG_NAME = "Offset"; //NON-NLS + private static final String INTERESTING_FILES_SET_TAG_NAME = "InterestingFileSset"; //NON-NLS private static final String ALERT_ATTRIBUTE = "alert"; //NON-NLS private static final String ENCODING_FOR_XML_FILE = "UTF-8"; //NON-NLS private static final String ASCII_ENCODING = "US-ASCII"; //NON-NLS @@ -135,7 +135,7 @@ final class UserDefinedFileTypesManager { /** * Create a file type that should match $MBR in Small2 image. */ - FileType fileType = new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), "predefinedRAW", true); + FileType fileType = new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), "Suspicious", true); this.addPredefinedFileType(fileType); /** @@ -143,7 +143,7 @@ final class UserDefinedFileTypesManager { */ // RJCTODO: Remove test file type. try { - fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), "predefinedASCII", true); + fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), "Benign", true); this.addPredefinedFileType(fileType); } catch (UnsupportedEncodingException ex) { UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS @@ -422,8 +422,8 @@ final class UserDefinedFileTypesManager { */ private static List readFileTypes(String filePath) throws IOException, ParserConfigurationException, SAXException { List fileTypes = new ArrayList<>(); - Path schemaFilePath = Paths.get(PlatformUtil.getUserConfigDirectory(), UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); - Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.XmlReader.class, schemaFilePath.toAbsolutePath().toString()); +// Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.XmlReader.class, UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); RJCTODO + Document doc = XMLUtil.loadDocument(filePath); if (doc != null) { Element fileTypesElem = doc.getDocumentElement(); if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME)) { From 2de508e11a2fe21348aab5706ca0df367f2212ac Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Dec 2014 08:35:43 -0500 Subject: [PATCH 73/84] Commit version of file types manager with fine grained-control, simpler will be better --- .../{UserDefinedFileTypes.xsd => FileTypeDefinitions.xsd} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Core/src/org/sleuthkit/autopsy/modules/filetypeid/{UserDefinedFileTypes.xsd => FileTypeDefinitions.xsd} (100%) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd similarity index 100% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypes.xsd rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd From 3562152a990706d2f9f6c79c5db8913b54b8785a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Dec 2014 08:36:05 -0500 Subject: [PATCH 74/84] Bug fixes for user-defined file types --- .../filetypeid/FileTypeDefinitions.xsd | 4 ++-- .../UserDefinedFileTypesManager.java | 24 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd index 727d86bb97..f0ec8fa6c4 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd @@ -26,7 +26,7 @@ - + @@ -42,7 +42,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index 74762cbbfb..c07607c41d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -118,12 +118,6 @@ final class UserDefinedFileTypesManager { * signature, and optional membership in an interesting files set. */ private UserDefinedFileTypesManager() { - /** - * Load the predefined types first so that they can be overwritten by - * any user-defined types with the same names. - */ - loadPredefinedFileTypes(); - loadUserDefinedFileTypes(); } /** @@ -235,6 +229,13 @@ final class UserDefinedFileTypesManager { * @return A mapping of file type names to file types, possibly empty. */ synchronized Map getFileTypes() { + /** + * Load the predefined types first so that they can be overwritten by + * any user-defined types with the same names. + */ + loadPredefinedFileTypes(); + loadUserDefinedFileTypes(); + /** * It is safe to return references to the internal file type objects * because they are immutable. @@ -248,6 +249,13 @@ final class UserDefinedFileTypesManager { * @return A mapping of file type names to file types, possibly empty. */ synchronized Map getUserDefinedFileTypes() { + /** + * Load the predefined types first so that they can be overwritten by + * any user-defined types with the same names. + */ + loadPredefinedFileTypes(); + loadUserDefinedFileTypes(); + /** * It is safe to return references to the internal file type objects * because they are immutable. @@ -422,8 +430,8 @@ final class UserDefinedFileTypesManager { */ private static List readFileTypes(String filePath) throws IOException, ParserConfigurationException, SAXException { List fileTypes = new ArrayList<>(); -// Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.XmlReader.class, UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); RJCTODO - Document doc = XMLUtil.loadDocument(filePath); + Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.class, UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); +// Document doc = XMLUtil.loadDocument(filePath); if (doc != null) { Element fileTypesElem = doc.getDocumentElement(); if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME)) { From acf83debc5bc7796d070b06882c6a6c943962673 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Dec 2014 17:07:37 -0500 Subject: [PATCH 75/84] Bug fixes for user-defined file types --- .../autopsy/coreutils/PlatformUtil.java | 38 ++- .../modules/filetypeid/Bundle.properties | 13 +- .../FileTypeIdGlobalSettingsPanel.form | 136 +++++---- .../FileTypeIdGlobalSettingsPanel.java | 258 +++++++++------- .../FileTypeIdIngestJobSettingsPanel.form | 19 +- .../FileTypeIdIngestJobSettingsPanel.java | 27 +- .../filetypeid/FileTypeIdIngestModule.java | 23 +- .../filetypeid/FileTypeIdModuleSettings.java | 28 +- ...{FileTypeDefinitions.xsd => FileTypes.xsd} | 12 +- .../UserDefinedFileTypeIdentifier.java | 16 +- .../UserDefinedFileTypesManager.java | 287 +++++++++--------- 11 files changed, 487 insertions(+), 370 deletions(-) rename Core/src/org/sleuthkit/autopsy/modules/filetypeid/{FileTypeDefinitions.xsd => FileTypes.xsd} (81%) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java index a5c01c9d42..da0423735b 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java @@ -30,6 +30,8 @@ import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryUsage; import java.nio.charset.Charset; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -211,37 +213,31 @@ public class PlatformUtil { * * @param resourceClass class in the same package as the resourceFile to * extract - * @param resourceFile resource file name to extract + * @param resourceFileName Name of the resource file to extract * @param overWrite true to overwrite an existing resource * @return true if extracted, false otherwise (if file already exists) * @throws IOException exception thrown if extract the file failed for IO * reasons */ - public static boolean extractResourceToUserConfigDir(final Class resourceClass, final String resourceFile, boolean overWrite) throws IOException { - final File userDir = new File(getUserConfigDirectory()); - - final File resourceFileF = new File(userDir + File.separator + resourceFile); - if (resourceFileF.exists() && !overWrite) { + public static boolean extractResourceToUserConfigDir(final Class resourceClass, final String resourceFileName, boolean overWrite) throws IOException { + Path resourceFilePath = Paths.get(getUserConfigDirectory(), resourceFileName); + final File resourceFile = resourceFilePath.toFile(); + if (resourceFile.exists() && !overWrite) { return false; } - resourceFileF.getParentFile().mkdirs(); + InputStream inputStream = resourceClass.getResourceAsStream(resourceFileName); + if (null == inputStream) { + return false; + } - InputStream inputStream = resourceClass.getResourceAsStream(resourceFile); - - OutputStream out = null; + resourceFile.getParentFile().mkdirs(); try (InputStream in = new BufferedInputStream(inputStream)) { - - OutputStream outFile = new FileOutputStream(resourceFileF); - out = new BufferedOutputStream(outFile); - int readBytes; - while ((readBytes = in.read()) != -1) { - out.write(readBytes); - } - } finally { - if (out != null) { - out.flush(); - out.close(); + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(resourceFile))) { + int readBytes; + while ((readBytes = in.read()) != -1) { + out.write(readBytes); + } } } return true; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index fbf61da330..b0dc119618 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -10,7 +10,6 @@ FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.toolTipText=Depending on how FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) FileTypeIdGlobalSettingsPanel.hexPrefixLabel.text=0x FileTypeIdGlobalSettingsPanel.deleteTypeButton.text=DeleteType -FileTypeIdGlobalSettingsPanel.newTypeButton.text=New Type FileTypeIdGlobalSettingsPanel.offsetTextField.text= FileTypeIdGlobalSettingsPanel.offsetLabel.text=Offset FileTypeIdGlobalSettingsPanel.postHitCheckBox.text=Post interesting file hit when found @@ -28,10 +27,18 @@ FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message=Signature is FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title=Missing Signature FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message=Offset must be a positive integer. FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title=Invalid Offset -FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message=The signature must be able to be converted to UTF-8 bytes. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidRawSignatureBytes.message=The signature has one or more invalid hexadecimal digits. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidAsciiSignatureBytes.message=The signature must be able to be converted to UTF-8 bytes. FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Signature +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.message=Interesting files set name is required if alert is requests. +FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.title=Missing Interesting Files Set Name FileTypeIdGlobalSettingsPanel.filesSetNameLabel.text=Files Set Name FileTypeIdGlobalSettingsPanel.filesSetNameTextField.text= FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title=Save Failed +FileTypeIdGlobalSettingsPanel.JOptionPane.loadFailed.title=Load Failed FileTypeIdGlobalSettingsPanel.hintTextArea.text=Enter a MIME type and signature to be used to identify files of that type. If the signature is a byte sequence, enter the sequence using two hex values for each byte, e.g., EEF0 is a two byte signature. -FileTypeIdGlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to file type definitions when ingest is running +FileTypeIdGlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to file type definitions when ingest is running! +UserDefinedFileTypesManager.loadFileTypes.errorMessage=Failed to load existing file type definitions. +UserDefinedFileTypesManager.saveFileTypes.errorMessage=Failed to save file type definitions. +FileTypeIdIngestJobSettingsPanel.skipSmallFilesCheckBox.text=Skip files smaller than {0} bytes +FileTypeIdGlobalSettingsPanel.newTypeButton.text=New Type diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index 8c04fa7747..bf19d17803 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -1,6 +1,14 @@

    + + + + + + + + @@ -17,7 +25,7 @@ - + @@ -25,31 +33,22 @@ + - - - - - - - - - + + + - + - - - - + - @@ -73,7 +72,20 @@ - + + + + + + + + + + + + + + @@ -83,55 +95,59 @@ - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - + @@ -149,6 +165,9 @@ + + + @@ -280,6 +299,9 @@ + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index 83bc32d167..abc61ac884 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -33,12 +33,12 @@ import javax.swing.event.DocumentListener; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; import javax.xml.bind.DatatypeConverter; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.modules.filetypeid.FileType.Signature; +import org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException; /** * A panel to allow a user to make custom file type definitions. In addition to @@ -187,7 +187,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane boolean fileTypeIsSelected = typesList.getSelectedIndex() != -1; deleteTypeButton.setEnabled(!ingestIsRunning && fileTypeIsSelected); - + boolean requiredFieldsPopulated = !mimeTypeTextField.getText().isEmpty() && !offsetTextField.getText().isEmpty() @@ -203,10 +203,18 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane */ @Override public void load() { - fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); - updateFileTypesListModel(); - if (!typesListModel.isEmpty()) { - typesList.setSelectedIndex(0); + try { + fileTypes = UserDefinedFileTypesManager.getInstance().getUserDefinedFileTypes(); + updateFileTypesListModel(); + if (!typesListModel.isEmpty()) { + typesList.setSelectedIndex(0); + } + } catch (UserDefinedFileTypesException ex) { + JOptionPane.showMessageDialog(null, + ex.getLocalizedMessage(), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.loadFailed.title"), + JOptionPane.ERROR_MESSAGE); + fileTypes = Collections.emptyMap(); } enableButtons(); } @@ -238,6 +246,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane this.signatureTextField.setText(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); offsetTextField.setText(Long.toString(signature.getOffset())); postHitCheckBox.setSelected(fileType.alertOnMatch()); + filesSetNameTextField.setEnabled(postHitCheckBox.isSelected()); filesSetNameTextField.setText(fileType.getFilesSetName()); } enableButtons(); @@ -248,7 +257,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * the panel. */ private void clearTypeDetailsComponents() { - typesList.setSelectedIndex(-1); + typesList.clearSelection(); mimeTypeTextField.setText(""); //NON-NLS signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); hexPrefixLabel.setVisible(true); @@ -324,6 +333,10 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane filesSetNameTextField = new javax.swing.JTextField(); ingestRunningWarningLabel = new javax.swing.JLabel(); + setMaximumSize(new java.awt.Dimension(500, 300)); + setPreferredSize(new java.awt.Dimension(500, 300)); + + typesList.setMinimumSize(new java.awt.Dimension(200, 0)); typesScrollPane.setViewportView(typesList); separator.setOrientation(javax.swing.SwingConstants.VERTICAL); @@ -378,6 +391,7 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane hintTextArea.setRows(5); hintTextArea.setText(org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.hintTextArea.text")); // NOI18N hintTextArea.setWrapStyleWord(true); + hintTextArea.setPreferredSize(new java.awt.Dimension(164, 70)); hintScrollPane.setViewportView(hintTextArea); org.openide.awt.Mnemonics.setLocalizedText(postHitCheckBox, org.openide.util.NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.postHitCheckBox.text")); // NOI18N @@ -399,33 +413,26 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() - .addGap(26, 26, 26) + .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addComponent(ingestRunningWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGap(30, 30, 30)) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE) .addGroup(layout.createSequentialGroup() - .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)) - .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() - .addGap(0, 0, Short.MAX_VALUE) - .addComponent(newTypeButton) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(deleteTypeButton) - .addGap(9, 9, 9))) + .addGap(0, 97, Short.MAX_VALUE))) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(offsetLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 71, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(filesSetNameLabel)) + .addComponent(offsetLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 71, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 179, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) .addComponent(signatureLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) @@ -443,52 +450,69 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 160, javax.swing.GroupLayout.PREFERRED_SIZE))))) .addComponent(hintScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 260, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(postHitCheckBox) - .addComponent(saveTypeButton)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) + .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 179, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(newTypeButton, javax.swing.GroupLayout.PREFERRED_SIZE, 71, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(saveTypeButton)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addComponent(filesSetNameLabel) + .addGap(188, 188, 188))))) .addGap(29, 29, 29)))) ); + + layout.linkSize(javax.swing.SwingConstants.HORIZONTAL, new java.awt.Component[] {deleteTypeButton, newTypeButton, saveTypeButton}); + layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() - .addGap(16, 16, 16) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) - .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 281, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() - .addComponent(hintScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(18, 18, 18) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(mimeTypeLabel) - .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(hintScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(mimeTypeLabel) + .addComponent(mimeTypeTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(signatureTypeLabel) + .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(hexPrefixLabel) + .addComponent(signatureLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(offsetLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(postHitCheckBox) + .addGap(1, 1, 1) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(filesSetNameLabel) + .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(signatureTypeLabel) - .addComponent(signatureTypeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(signatureTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(hexPrefixLabel) - .addComponent(signatureLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(offsetTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(offsetLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(postHitCheckBox) - .addGap(8, 8, 8) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(filesSetNameLabel) - .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(saveTypeButton)) - .addGroup(layout.createSequentialGroup() - .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 249, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(deleteTypeButton) - .addComponent(newTypeButton)))) + .addComponent(newTypeButton) + .addComponent(saveTypeButton))) + .addGroup(layout.createSequentialGroup() + .addGap(11, 11, 11) + .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 257, javax.swing.GroupLayout.PREFERRED_SIZE))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(ingestRunningWarningLabel) - .addContainerGap(22, Short.MAX_VALUE)) + .addContainerGap(18, Short.MAX_VALUE)) ); + + layout.linkSize(javax.swing.SwingConstants.VERTICAL, new java.awt.Component[] {deleteTypeButton, newTypeButton, saveTypeButton}); + }// //GEN-END:initComponents private void newTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_newTypeButtonActionPerformed @@ -498,82 +522,90 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane private void deleteTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteTypeButtonActionPerformed String typeName = typesList.getSelectedValue(); fileTypes.remove(typeName); - clearTypeDetailsComponents(); + updateFileTypesListModel(); + if (!typesListModel.isEmpty()) { + typesList.setSelectedIndex(0); + } }//GEN-LAST:event_deleteTypeButtonActionPerformed private void saveTypeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_saveTypeButtonActionPerformed - try { - /** - * Get the MIME type. - */ - String typeName = mimeTypeTextField.getText(); - if (typeName.isEmpty()) { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.title"), - JOptionPane.ERROR_MESSAGE); - return; - } + /** + * Get the MIME type. + */ + String typeName = mimeTypeTextField.getText(); + if (typeName.isEmpty()) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidMIMEType.title"), + JOptionPane.ERROR_MESSAGE); + return; + } - /** - * Get the signature type. - */ - FileType.Signature.Type sigType = signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; + /** + * Get the signature type. + */ + FileType.Signature.Type sigType = signatureTypeComboBox.getSelectedItem() == FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM ? FileType.Signature.Type.RAW : FileType.Signature.Type.ASCII; - /** - * Get the signature bytes. - */ - String sigString = signatureTextField.getText(); - if (sigString.isEmpty()) { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title"), - JOptionPane.ERROR_MESSAGE); - return; - } - byte[] signatureBytes; - if (FileType.Signature.Type.RAW == sigType) { + /** + * Get the signature bytes. + */ + String sigString = signatureTextField.getText(); + if (sigString.isEmpty()) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title"), + JOptionPane.ERROR_MESSAGE); + return; + } + byte[] signatureBytes; + if (FileType.Signature.Type.RAW == sigType) { + try { signatureBytes = DatatypeConverter.parseHexBinary(sigString); - } else { - signatureBytes = sigString.getBytes(Charset.forName("UTF-8")); - } - - /** - * Get the offset. - */ - long offset = Long.parseUnsignedLong(offsetTextField.getText()); - - /** - * Get the interesting files set details. - */ - String filesSetName = filesSetNameTextField.getText(); - if (postHitCheckBox.isSelected() && filesSetName.isEmpty()) { + } catch (IllegalArgumentException ex) { JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidRawSignatureBytes.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title"), JOptionPane.ERROR_MESSAGE); + return; } + } else { + signatureBytes = sigString.getBytes(Charset.forName("UTF-8")); + } - /** - * Put it all together and reset the file types list component. - */ - FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); - FileType fileType = new FileType(typeName, signature, filesSetName, postHitCheckBox.isSelected()); - fileTypes.put(typeName, fileType); - updateFileTypesListModel(); - typesList.setSelectedValue(fileType.getMimeType(), true); - + /** + * Get the offset. + */ + long offset; + try { + offset = Long.parseUnsignedLong(offsetTextField.getText()); } catch (NumberFormatException ex) { JOptionPane.showMessageDialog(null, NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message"), NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title"), JOptionPane.ERROR_MESSAGE); - } catch (IllegalArgumentException ex) { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.message"), - NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title"), - JOptionPane.ERROR_MESSAGE); + return; } + + /** + * Get the interesting files set details. + */ + String filesSetName = filesSetNameTextField.getText(); + if (postHitCheckBox.isSelected() && filesSetName.isEmpty()) { + JOptionPane.showMessageDialog(null, + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.message"), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.title"), + JOptionPane.ERROR_MESSAGE); + return; + } + + /** + * Put it all together and reset the file types list component. + */ + FileType.Signature signature = new FileType.Signature(signatureBytes, offset, sigType); + FileType fileType = new FileType(typeName, signature, filesSetName, postHitCheckBox.isSelected()); + fileTypes.put(typeName, fileType); + updateFileTypesListModel(); + typesList.setSelectedValue(fileType.getMimeType(), true); }//GEN-LAST:event_saveTypeButtonActionPerformed private void postHitCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_postHitCheckBoxActionPerformed diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form index 7a24e05f05..f03a25070c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.form @@ -18,7 +18,10 @@ - + + + + @@ -28,7 +31,9 @@ - + + + @@ -48,5 +53,15 @@ + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java index ff65c1ac63..87dd8dd53e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestJobSettingsPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; @@ -49,8 +50,10 @@ class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPane */ private void customizeComponents() { skipKnownCheckBox.setSelected(settings.skipKnownFiles()); + skipSmallFilesCheckBox.setSelected(settings.skipSmallFiles()); + skipSmallFilesCheckBox.setText(NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipSmallFilesCheckBox.text", settings.minFileSizeInBytes())); } - + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -61,6 +64,7 @@ class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPane private void initComponents() { skipKnownCheckBox = new javax.swing.JCheckBox(); + skipSmallFilesCheckBox = new javax.swing.JCheckBox(); skipKnownCheckBox.setSelected(true); skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipKnownCheckBox.text")); // NOI18N @@ -71,13 +75,22 @@ class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPane } }); + skipSmallFilesCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdIngestJobSettingsPanel.class, "FileTypeIdIngestJobSettingsPanel.skipSmallFilesCheckBox.text")); // NOI18N + skipSmallFilesCheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + skipSmallFilesCheckBoxActionPerformed(evt); + } + }); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) - .addComponent(skipKnownCheckBox) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(skipSmallFilesCheckBox) + .addComponent(skipKnownCheckBox)) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); layout.setVerticalGroup( @@ -85,14 +98,22 @@ class FileTypeIdIngestJobSettingsPanel extends IngestModuleIngestJobSettingsPane .addGroup(layout.createSequentialGroup() .addGap(11, 11, 11) .addComponent(skipKnownCheckBox) - .addContainerGap(47, Short.MAX_VALUE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(skipSmallFilesCheckBox) + .addContainerGap(60, Short.MAX_VALUE)) ); }// //GEN-END:initComponents private void skipKnownCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipKnownCheckBoxActionPerformed settings.setSkipKnownFiles(skipKnownCheckBox.isSelected()); }//GEN-LAST:event_skipKnownCheckBoxActionPerformed + + private void skipSmallFilesCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipSmallFilesCheckBoxActionPerformed + settings.setSkipSmallFiles(skipSmallFilesCheckBox.isSelected()); + }//GEN-LAST:event_skipSmallFilesCheckBoxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JCheckBox skipKnownCheckBox; + private javax.swing.JCheckBox skipSmallFilesCheckBox; // End of variables declaration//GEN-END:variables } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 6d6a1796e5..2759180c26 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -20,8 +20,10 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.util.HashMap; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; @@ -43,7 +45,6 @@ import org.sleuthkit.datamodel.BlackboardAttribute; public class FileTypeIdIngestModule implements FileIngestModule { private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); - private static final long MIN_FILE_SIZE = 512; private final FileTypeIdModuleSettings settings; private long jobId; private static final HashMap totalsForIngestJobs = new HashMap<>(); @@ -76,7 +77,13 @@ public class FileTypeIdIngestModule implements FileIngestModule { */ FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { this.settings = settings; - this.userDefinedFileTypeIdentifier = new UserDefinedFileTypeIdentifier(); + userDefinedFileTypeIdentifier = new UserDefinedFileTypeIdentifier(); + try { + userDefinedFileTypeIdentifier.loadFileTypes(); + } catch (UserDefinedFileTypesManager.UserDefinedFileTypesException ex) { + logger.log(Level.SEVERE, "Failed to load file types", ex); + MessageNotifyUtil.Notify.error(FileTypeIdModuleFactory.getModuleName(), ex.getMessage()); + } } /** @@ -112,11 +119,10 @@ public class FileTypeIdIngestModule implements FileIngestModule { /** * Filter out very small files to minimize false positives. */ - // RJCTODO: Make this size a setting -// if (file.getSize() < MIN_FILE_SIZE) { -// return ProcessResult.OK; -// } - + if (settings.skipSmallFiles() && file.getSize() < settings.minFileSizeInBytes()) { + return ProcessResult.OK; + } + try { long startTime = System.currentTimeMillis(); FileType fileType = this.userDefinedFileTypeIdentifier.identify(file); @@ -133,7 +139,8 @@ public class FileTypeIdIngestModule implements FileIngestModule { /** * Use the MIME type as the category, i.e., the rule that - * determined this file belongs to the interesting files set. + * determined this file belongs to the interesting files + * set. */ BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY.getTypeID(), moduleName, fileType.getMimeType()); artifact.addAttribute(ruleNameAttribute); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java index ddf5371045..2b774451a5 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java @@ -27,23 +27,26 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { private static final long serialVersionUID = 1L; - private volatile boolean skipKnownFiles = true; + private static final long MIN_FILE_SIZE_IN_BYTES = 512; + private boolean skipKnownFiles = true; + private boolean skipSmallFiles = true; FileTypeIdModuleSettings() { } - FileTypeIdModuleSettings(boolean skipKnownFiles) { + FileTypeIdModuleSettings(boolean skipKnownFiles, boolean skipSmallFiles) { this.skipKnownFiles = skipKnownFiles; + this.skipSmallFiles = skipSmallFiles; } - + /** * @inheritDoc */ @Override public long getVersionNumber() { return serialVersionUID; - } - + } + void setSkipKnownFiles(boolean enabled) { skipKnownFiles = enabled; } @@ -51,4 +54,17 @@ public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { boolean skipKnownFiles() { return skipKnownFiles; } -} \ No newline at end of file + + void setSkipSmallFiles(boolean enabled) { + this.skipKnownFiles = enabled; + } + + boolean skipSmallFiles() { + return skipSmallFiles; + } + + long minFileSizeInBytes() { + return MIN_FILE_SIZE_IN_BYTES; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypes.xsd similarity index 81% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypes.xsd index f0ec8fa6c4..26aa720ff1 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDefinitions.xsd +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypes.xsd @@ -5,7 +5,7 @@ - + @@ -26,23 +26,23 @@ - + - + - + - + - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java index c93cc9a547..f756785585 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.util.HashMap; import java.util.Map; import org.sleuthkit.datamodel.AbstractFile; @@ -26,14 +27,25 @@ import org.sleuthkit.datamodel.AbstractFile; */ final class UserDefinedFileTypeIdentifier { - private final Map fileTypes; + private Map fileTypes; /** * Creates an object that can do file type identification for user-defined * file types. */ UserDefinedFileTypeIdentifier() { - this.fileTypes = UserDefinedFileTypesManager.getInstance().getFileTypes(); + fileTypes = new HashMap<>(); + } + + /** + * Gets the user-defined file types from the user-defined file types + * manager. + * + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + void loadFileTypes() throws UserDefinedFileTypesManager.UserDefinedFileTypesException { + fileTypes = UserDefinedFileTypesManager.getInstance().getFileTypes(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java index c07607c41d..293ac805bd 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypesManager.java @@ -36,6 +36,7 @@ import org.w3c.dom.Element; import org.w3c.dom.NodeList; import javax.xml.bind.DatatypeConverter; import javax.xml.transform.TransformerException; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.XMLUtil; @@ -50,18 +51,17 @@ import org.xml.sax.SAXException; * user-defined file types as a complete set - there is no concept of adding, * editing or deleting file types singly. This works because this class is not * exposed outside of this ingest module package and is ONLY used in a very - * specific paradigm where there is a single modal writer of file types in the - * form of a global settings panel that disables itself when ingest is running - * so that multiple readers in the form of file ingest modules get a consistent - * set of file type definitions. Moreover, there is no enforcement of compliance - * with this paradigm by this class - it merely participates in the scheme. + * specific paradigm. In this paradigm, there is a single modal writer of file + * types in the form of a global settings panel that disables itself when ingest + * is running so that multiple readers in the form of file ingest modules get a + * consistent set of file type definitions. *

    * Thread-safe. */ final class UserDefinedFileTypesManager { private static final Logger logger = Logger.getLogger(UserDefinedFileTypesManager.class.getName()); - private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypeDefinitions.xsd"; //NON-NLS + private static final String FILE_TYPE_DEFINITIONS_SCHEMA_FILE = "FileTypes.xsd"; //NON-NLS private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.xml"; //NON-NLS private static final String FILE_TYPES_TAG_NAME = "FileTypes"; //NON-NLS private static final String FILE_TYPE_TAG_NAME = "FileType"; //NON-NLS @@ -76,13 +76,6 @@ final class UserDefinedFileTypesManager { private static final String ASCII_ENCODING = "US-ASCII"; //NON-NLS private static UserDefinedFileTypesManager instance; - /** - * Predefined file types are stored in this mapping of MIME types to file - * types. Access to this map is guarded by the intrinsic lock of the - * user-defined file types manager for thread-safety. - */ - private final Map predefinedFileTypes = new HashMap<>(); - /** * File types to be persisted to the user-defined file type definitions file * are stored in this mapping of MIME types to file types. Access to this @@ -101,16 +94,17 @@ final class UserDefinedFileTypesManager { private final Map fileTypes = new HashMap<>(); /** - * Gets the manager of user-defined file types characterized by MIME type, - * signature, and optional membership in an interesting files set. + * Gets the singleton manager of user-defined file types characterized by + * MIME type, signature, and optional membership in an interesting files + * set. * * @return The user-defined file types manager singleton. */ synchronized static UserDefinedFileTypesManager getInstance() { - if (UserDefinedFileTypesManager.instance == null) { - UserDefinedFileTypesManager.instance = new UserDefinedFileTypesManager(); + if (instance == null) { + instance = new UserDefinedFileTypesManager(); } - return UserDefinedFileTypesManager.instance; + return instance; } /** @@ -120,29 +114,70 @@ final class UserDefinedFileTypesManager { private UserDefinedFileTypesManager() { } + /** + * Gets both the predefined and the user-defined file types. + * + * @return A mapping of file type names to file types, possibly empty. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized Map getFileTypes() throws UserDefinedFileTypesException { + loadFileTypes(); + + /** + * It is safe to return references to the internal file type objects + * because they are immutable. Note that + * Collections.unmodifiableCollection() is not used here because this + * view of the file types is a snapshot. + */ + return new HashMap<>(fileTypes); + } + + /** + * Gets the user-defined file types. + * + * @return A mapping of file type names to file types, possibly empty. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + synchronized Map getUserDefinedFileTypes() throws UserDefinedFileTypesException { + loadFileTypes(); + + /** + * It is safe to return references to the internal file type objects + * because they are immutable. Note that + * Collections.unmodifiableCollection() is not used here because this + * view of the file types is a snapshot. + */ + return new HashMap<>(userDefinedFileTypes); + } + + /** + * Loads the MIME type to file type mappings with predefined and + * user-defined types. + * + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + private void loadFileTypes() throws UserDefinedFileTypesException { + fileTypes.clear(); + userDefinedFileTypes.clear(); + /** + * Load the predefined types first so that they can be overwritten by + * any user-defined types with the same names. + */ + loadPredefinedFileTypes(); + loadUserDefinedFileTypes(); + } + /** * Adds the predefined file types to the in-memory mappings of MIME types to * file types. + * + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException */ - private void loadPredefinedFileTypes() { - // RJCTODO: Remove test file type. - /** - * Create a file type that should match $MBR in Small2 image. - */ - FileType fileType = new FileType("predefinedRAW", new Signature(new byte[]{(byte) 0x66, (byte) 0x73, (byte) 0x00}, 8L, FileType.Signature.Type.RAW), "Suspicious", true); - this.addPredefinedFileType(fileType); - - /** - * Create a file type that should match test.txt in the Small2 image. - */ - // RJCTODO: Remove test file type. - try { - fileType = new FileType("predefinedASCII", new Signature("hello".getBytes(UserDefinedFileTypesManager.ASCII_ENCODING), 0L, FileType.Signature.Type.ASCII), "Benign", true); - this.addPredefinedFileType(fileType); - } catch (UnsupportedEncodingException ex) { - UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Unable to create 'predefinedASCII' predefined file type definition", ex); //NON-NLS - } - + private void loadPredefinedFileTypes() throws UserDefinedFileTypesException { try { // RJCTODO: Remove this code from TikaFileTypeDetector.java. // try { @@ -166,39 +201,32 @@ final class UserDefinedFileTypesManager { // catch (IndexOutOfBoundsException e) { // // do nothing // } - fileType = new FileType("text/xml", new Signature(" getFileTypes() { - /** - * Load the predefined types first so that they can be overwritten by - * any user-defined types with the same names. - */ - loadPredefinedFileTypes(); - loadUserDefinedFileTypes(); - - /** - * It is safe to return references to the internal file type objects - * because they are immutable. - */ - return new HashMap<>(this.fileTypes); - } - - /** - * Gets the user-defined file types. - * - * @return A mapping of file type names to file types, possibly empty. - */ - synchronized Map getUserDefinedFileTypes() { - /** - * Load the predefined types first so that they can be overwritten by - * any user-defined types with the same names. - */ - loadPredefinedFileTypes(); - loadUserDefinedFileTypes(); - - /** - * It is safe to return references to the internal file type objects - * because they are immutable. - */ - return new HashMap<>(this.userDefinedFileTypes); + userDefinedFileTypes.put(fileType.getMimeType(), fileType); + fileTypes.put(fileType.getMimeType(), fileType); } /** @@ -269,35 +257,15 @@ final class UserDefinedFileTypesManager { * @param newFileTypes A mapping of file type names to user-defined file * types. */ - synchronized void setUserDefinedFileTypes(Map newFileTypes) throws UserDefinedFileTypesManager.UserDefinedFileTypesException { + synchronized void setUserDefinedFileTypes(Map newFileTypes) throws UserDefinedFileTypesException { try { - String filePath = UserDefinedFileTypesManager.getFileTypeDefinitionsFilePath(UserDefinedFileTypesManager.USER_DEFINED_TYPE_DEFINITIONS_FILE); - UserDefinedFileTypesManager.XmlWriter.writeFileTypes(newFileTypes.values(), filePath); - + String filePath = getFileTypeDefinitionsFilePath(USER_DEFINED_TYPE_DEFINITIONS_FILE); + XmlWriter.writeFileTypes(newFileTypes.values(), filePath); } catch (ParserConfigurationException | FileNotFoundException | UnsupportedEncodingException | TransformerException ex) { - UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); - throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); + throwUserDefinedFileTypesException(ex, "UserDefinedFileTypesManager.saveFileTypes.errorMessage"); } catch (IOException ex) { - UserDefinedFileTypesManager.logger.log(Level.SEVERE, "Failed to write file types file", ex); - throw new UserDefinedFileTypesManager.UserDefinedFileTypesException(ex.getLocalizedMessage()); + throwUserDefinedFileTypesException(ex, "UserDefinedFileTypesManager.saveFileTypes.errorMessage"); } - - /** - * Clear and reinitialize the user-defined file type map. It is safe to - * hold references to file type objects obtained for the caller because - * they are immutable. - */ - this.userDefinedFileTypes.clear(); - this.userDefinedFileTypes.putAll(newFileTypes); - - /** - * Clear and reinitialize the combined file type map, loading the - * predefined types first so that they can be overwritten by any - * user-defined types with the same names. - */ - this.fileTypes.clear(); - this.fileTypes.putAll(this.predefinedFileTypes); - this.fileTypes.putAll(this.userDefinedFileTypes); } /** @@ -330,13 +298,13 @@ final class UserDefinedFileTypesManager { */ private static void writeFileTypes(Collection fileTypes, String filePath) throws ParserConfigurationException, IOException, FileNotFoundException, UnsupportedEncodingException, TransformerException { Document doc = XMLUtil.createDocument(); - Element fileTypesElem = doc.createElement(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME); + Element fileTypesElem = doc.createElement(FILE_TYPES_TAG_NAME); doc.appendChild(fileTypesElem); for (FileType fileType : fileTypes) { - Element fileTypeElem = UserDefinedFileTypesManager.XmlWriter.createFileTypeElement(fileType, doc); + Element fileTypeElem = XmlWriter.createFileTypeElement(fileType, doc); fileTypesElem.appendChild(fileTypeElem); } - XMLUtil.saveDocument(doc, UserDefinedFileTypesManager.ENCODING_FOR_XML_FILE, filePath); + XMLUtil.saveDocument(doc, ENCODING_FOR_XML_FILE, filePath); } /** @@ -347,7 +315,7 @@ final class UserDefinedFileTypesManager { * @return An XML element. */ private static Element createFileTypeElement(FileType fileType, Document doc) { - Element fileTypeElem = doc.createElement(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); + Element fileTypeElem = doc.createElement(FILE_TYPE_TAG_NAME); XmlWriter.addMimeTypeElement(fileType, fileTypeElem, doc); XmlWriter.addSignatureElement(fileType, fileTypeElem, doc); XmlWriter.addInterestingFilesSetElement(fileType, fileTypeElem, doc); @@ -363,7 +331,7 @@ final class UserDefinedFileTypesManager { * @param doc The WC3 DOM object to use to create the XML. */ private static void addMimeTypeElement(FileType fileType, Element fileTypeElem, Document doc) { - Element typeNameElem = doc.createElement(UserDefinedFileTypesManager.MIME_TYPE_TAG_NAME); + Element typeNameElem = doc.createElement(MIME_TYPE_TAG_NAME); typeNameElem.setTextContent(fileType.getMimeType()); fileTypeElem.appendChild(typeNameElem); } @@ -377,17 +345,17 @@ final class UserDefinedFileTypesManager { */ private static void addSignatureElement(FileType fileType, Element fileTypeElem, Document doc) { Signature signature = fileType.getSignature(); - Element signatureElem = doc.createElement(UserDefinedFileTypesManager.SIGNATURE_TAG_NAME); + Element signatureElem = doc.createElement(SIGNATURE_TAG_NAME); - Element bytesElem = doc.createElement(UserDefinedFileTypesManager.BYTES_TAG_NAME); + Element bytesElem = doc.createElement(BYTES_TAG_NAME); bytesElem.setTextContent(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); signatureElem.appendChild(bytesElem); - Element offsetElem = doc.createElement(UserDefinedFileTypesManager.OFFSET_TAG_NAME); + Element offsetElem = doc.createElement(OFFSET_TAG_NAME); offsetElem.setTextContent(DatatypeConverter.printLong(signature.getOffset())); signatureElem.appendChild(offsetElem); - signatureElem.setAttribute(UserDefinedFileTypesManager.SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); + signatureElem.setAttribute(SIGNATURE_TYPE_ATTRIBUTE, signature.getType().toString()); fileTypeElem.appendChild(signatureElem); } @@ -399,9 +367,11 @@ final class UserDefinedFileTypesManager { * @param doc The WC3 DOM object to use to create the XML. */ private static void addInterestingFilesSetElement(FileType fileType, Element fileTypeElem, Document doc) { - Element filesSetElem = doc.createElement(UserDefinedFileTypesManager.INTERESTING_FILES_SET_TAG_NAME); - filesSetElem.setTextContent(fileType.getFilesSetName()); - fileTypeElem.appendChild(filesSetElem); + if (!fileType.getFilesSetName().isEmpty()) { + Element filesSetElem = doc.createElement(INTERESTING_FILES_SET_TAG_NAME); + filesSetElem.setTextContent(fileType.getFilesSetName()); + fileTypeElem.appendChild(filesSetElem); + } } /** @@ -411,7 +381,7 @@ final class UserDefinedFileTypesManager { * @param fileTypeElem The parent file type element. */ private static void addAlertAttribute(FileType fileType, Element fileTypeElem) { - fileTypeElem.setAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); + fileTypeElem.setAttribute(ALERT_ATTRIBUTE, Boolean.toString(fileType.alertOnMatch())); } } @@ -430,12 +400,11 @@ final class UserDefinedFileTypesManager { */ private static List readFileTypes(String filePath) throws IOException, ParserConfigurationException, SAXException { List fileTypes = new ArrayList<>(); - Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.class, UserDefinedFileTypesManager.FILE_TYPE_DEFINITIONS_SCHEMA_FILE); -// Document doc = XMLUtil.loadDocument(filePath); + Document doc = XMLUtil.loadDocument(filePath, UserDefinedFileTypesManager.class, FILE_TYPE_DEFINITIONS_SCHEMA_FILE); if (doc != null) { Element fileTypesElem = doc.getDocumentElement(); - if (fileTypesElem != null && fileTypesElem.getNodeName().equals(UserDefinedFileTypesManager.FILE_TYPES_TAG_NAME)) { - NodeList fileTypeElems = fileTypesElem.getElementsByTagName(UserDefinedFileTypesManager.FILE_TYPE_TAG_NAME); + if (fileTypesElem != null && fileTypesElem.getNodeName().equals(FILE_TYPES_TAG_NAME)) { + NodeList fileTypeElems = fileTypesElem.getElementsByTagName(FILE_TYPE_TAG_NAME); for (int i = 0; i < fileTypeElems.getLength(); ++i) { Element fileTypeElem = (Element) fileTypeElems.item(i); FileType fileType = XmlReader.parseFileType(fileTypeElem); @@ -469,7 +438,7 @@ final class UserDefinedFileTypesManager { * @return A MIME type string. */ private static String parseMimeType(Element fileTypeElem) { - return getChildElementTextContent(fileTypeElem, UserDefinedFileTypesManager.MIME_TYPE_TAG_NAME); + return getChildElementTextContent(fileTypeElem, MIME_TYPE_TAG_NAME); } /** @@ -479,16 +448,16 @@ final class UserDefinedFileTypesManager { * @return The signature. */ private static Signature parseSignature(Element fileTypeElem) throws IllegalArgumentException, NumberFormatException { - NodeList signatureElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypesManager.SIGNATURE_TAG_NAME); + NodeList signatureElems = fileTypeElem.getElementsByTagName(SIGNATURE_TAG_NAME); Element signatureElem = (Element) signatureElems.item(0); - String sigTypeAttribute = signatureElem.getAttribute(UserDefinedFileTypesManager.SIGNATURE_TYPE_ATTRIBUTE); + String sigTypeAttribute = signatureElem.getAttribute(SIGNATURE_TYPE_ATTRIBUTE); Signature.Type signatureType = Signature.Type.valueOf(sigTypeAttribute); - String sigBytesString = getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.BYTES_TAG_NAME); + String sigBytesString = getChildElementTextContent(signatureElem, BYTES_TAG_NAME); byte[] signatureBytes = DatatypeConverter.parseHexBinary(sigBytesString); - String offsetString = getChildElementTextContent(signatureElem, UserDefinedFileTypesManager.OFFSET_TAG_NAME); + String offsetString = getChildElementTextContent(signatureElem, OFFSET_TAG_NAME); long offset = DatatypeConverter.parseLong(offsetString); return new Signature(signatureBytes, offset, signatureType); @@ -502,7 +471,7 @@ final class UserDefinedFileTypesManager { */ private static String parseInterestingFilesSet(Element fileTypeElem) { String filesSetName = ""; - NodeList filesSetElems = fileTypeElem.getElementsByTagName(UserDefinedFileTypesManager.INTERESTING_FILES_SET_TAG_NAME); + NodeList filesSetElems = fileTypeElem.getElementsByTagName(INTERESTING_FILES_SET_TAG_NAME); if (filesSetElems.getLength() > 0) { Element filesSetElem = (Element) filesSetElems.item(0); filesSetName = filesSetElem.getTextContent(); @@ -517,7 +486,7 @@ final class UserDefinedFileTypesManager { * @return True or false; */ private static boolean parseAlert(Element fileTypeElem) { - String alertAttribute = fileTypeElem.getAttribute(UserDefinedFileTypesManager.ALERT_ATTRIBUTE); + String alertAttribute = fileTypeElem.getAttribute(ALERT_ATTRIBUTE); return Boolean.parseBoolean(alertAttribute); } @@ -536,6 +505,22 @@ final class UserDefinedFileTypesManager { } + /** + * Logs an exception, bundles the exception with a simple message in a + * uniform exception type, and throws the wrapper exception. + * + * @param ex The exception to wrap. + * @param messageKey A key into the bundle file that maps to the desired + * message. + * @throws + * org.sleuthkit.autopsy.modules.filetypeid.UserDefinedFileTypesManager.UserDefinedFileTypesException + */ + private void throwUserDefinedFileTypesException(Exception ex, String messageKey) throws UserDefinedFileTypesException { + String message = NbBundle.getMessage(UserDefinedFileTypesManager.class, messageKey); + logger.log(Level.SEVERE, message, ex); + throw new UserDefinedFileTypesException(message, ex); + } + /** * Used to translate more implementation-details-specific exceptions (which * are logged by this class) into more generic exceptions for propagation to @@ -546,6 +531,10 @@ final class UserDefinedFileTypesManager { UserDefinedFileTypesException(String message) { super(message); } + + UserDefinedFileTypesException(String message, Throwable throwable) { + super(message, throwable); + } } } From f1262af4872970368c7b8addbf15109025cd1aa4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Dec 2014 21:29:16 -0500 Subject: [PATCH 76/84] Bug fixes for user-defined file types --- .../modules/filetypeid/Bundle.properties | 1 - .../FileTypeIdGlobalSettingsPanel.form | 10 ++-- .../FileTypeIdGlobalSettingsPanel.java | 46 +++++++++++++++---- .../filetypeid/FileTypeIdIngestModule.java | 3 ++ .../filetypeid/FileTypeIdModuleSettings.java | 2 +- 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties index b0dc119618..ef33f85fc6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties @@ -28,7 +28,6 @@ FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignature.title=Missing Signatu FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.message=Offset must be a positive integer. FileTypeIdGlobalSettingsPanel.JOptionPane.invalidOffset.title=Invalid Offset FileTypeIdGlobalSettingsPanel.JOptionPane.invalidRawSignatureBytes.message=The signature has one or more invalid hexadecimal digits. -FileTypeIdGlobalSettingsPanel.JOptionPane.invalidAsciiSignatureBytes.message=The signature must be able to be converted to UTF-8 bytes. FileTypeIdGlobalSettingsPanel.JOptionPane.invalidSignatureBytes.title=Invalid Signature FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.message=Interesting files set name is required if alert is requests. FileTypeIdGlobalSettingsPanel.JOptionPane.invalidInterestingFilesSetName.title=Missing Interesting Files Set Name diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form index bf19d17803..c168205dad 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.form @@ -134,10 +134,12 @@ - - + - + + + + @@ -147,7 +149,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java index abc61ac884..d0847fc63f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdGlobalSettingsPanel.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.awt.EventQueue; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; +import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; @@ -33,6 +34,7 @@ import javax.swing.event.DocumentListener; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; import javax.xml.bind.DatatypeConverter; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestManager; @@ -61,10 +63,13 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane /** * This panel implements a property change listener that listens to ingest - * job events to disable the buttons on the panel if ingest is running. This - * is done to prevent changes to user-defined types while the type - * definitions are in use. + * job events so it can disable the buttons on the panel if ingest is + * running. This is done to prevent changes to user-defined types while the + * type definitions are in use. */ + // TODO: Disabling during ingest would not be necessary if the file ingest + // modules obtained and shared a per data source ingest job snapshot of the + // file type definitions. IngestJobEventPropertyChangeListener ingestJobEventsListener; /** @@ -107,6 +112,11 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane signatureTypeComboBox.setSelectedItem(FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM); } + /** + * Adds a listener to the types list component so that the components in the + * file type details section of the panel can be populated and cleared based + * on the selection in the list. + */ private void addTypeListSelectionListener() { typesList.addListSelectionListener(new ListSelectionListener() { @Override @@ -156,6 +166,9 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane * prevent changes to user-defined types while the type definitions are in * use. */ + // TODO: Disabling during ingest would not be necessary if the file ingest + // modules obtained and shared a per data source ingest job snapshot of the + // file type definitions. private void addIngestJobEventsListener() { ingestJobEventsListener = new IngestJobEventPropertyChangeListener(); IngestManager.getInstance().addIngestJobEventListener(ingestJobEventsListener); @@ -239,11 +252,25 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane String mimeType = typesList.getSelectedValue(); FileType fileType = fileTypes.get(mimeType); if (null != fileType) { - mimeTypeTextField.setText(mimeType); + mimeTypeTextField.setText(fileType.getMimeType()); Signature signature = fileType.getSignature(); FileType.Signature.Type sigType = signature.getType(); signatureTypeComboBox.setSelectedItem(sigType == FileType.Signature.Type.RAW ? FileTypeIdGlobalSettingsPanel.RAW_SIGNATURE_TYPE_COMBO_BOX_ITEM : FileTypeIdGlobalSettingsPanel.ASCII_SIGNATURE_TYPE_COMBO_BOX_ITEM); - this.signatureTextField.setText(DatatypeConverter.printHexBinary(signature.getSignatureBytes())); + String signatureBytes; + if (Signature.Type.RAW == signature.getType()) { + signatureBytes = DatatypeConverter.printHexBinary(signature.getSignatureBytes()); + } else { + try { + signatureBytes = new String(signature.getSignatureBytes(), "UTF-8"); + } catch (UnsupportedEncodingException ex) { + JOptionPane.showMessageDialog(null, + ex.getLocalizedMessage(), + NbBundle.getMessage(FileTypeIdGlobalSettingsPanel.class, "FileTypeIdGlobalSettingsPanel.JOptionPane.storeFailed.title"), + JOptionPane.ERROR_MESSAGE); + signatureBytes = ""; + } + } + signatureTextField.setText(signatureBytes); offsetTextField.setText(Long.toString(signature.getOffset())); postHitCheckBox.setSelected(fileType.alertOnMatch()); filesSetNameTextField.setEnabled(postHitCheckBox.isSelected()); @@ -499,16 +526,17 @@ final class FileTypeIdGlobalSettingsPanel extends IngestModuleGlobalSettingsPane .addComponent(filesSetNameTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))) .addComponent(typesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 219, javax.swing.GroupLayout.PREFERRED_SIZE)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(deleteTypeButton) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(newTypeButton) - .addComponent(saveTypeButton))) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(deleteTypeButton) + .addComponent(saveTypeButton)))) .addGroup(layout.createSequentialGroup() .addGap(11, 11, 11) .addComponent(separator, javax.swing.GroupLayout.PREFERRED_SIZE, 257, javax.swing.GroupLayout.PREFERRED_SIZE))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(ingestRunningWarningLabel) - .addContainerGap(18, Short.MAX_VALUE)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); layout.linkSize(javax.swing.SwingConstants.VERTICAL, new java.awt.Component[] {deleteTypeButton, newTypeButton, saveTypeButton}); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 2759180c26..c3c6bbcb5b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -100,6 +100,9 @@ public class FileTypeIdIngestModule implements FileIngestModule { */ @Override public ProcessResult process(AbstractFile file) { + + String name = file.getName(); + /** * Skip unallocated space and unused blocks files. */ diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java index 2b774451a5..270c604d25 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java @@ -56,7 +56,7 @@ public class FileTypeIdModuleSettings implements IngestModuleIngestJobSettings { } void setSkipSmallFiles(boolean enabled) { - this.skipKnownFiles = enabled; + this.skipSmallFiles = enabled; } boolean skipSmallFiles() { From 2dc73333cbb5c6f2dd69091b50f80207c21de3b7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Dec 2014 21:53:12 -0500 Subject: [PATCH 77/84] Remove XML test from tika file type detector --- .../sleuthkit/autopsy/casemodule/Case.java | 2 +- .../autopsy/ingest/IngestManager.java | 3 +-- .../filetypeid/TikaFileTypeDetector.java | 12 ---------- .../UserDefinedFileTypesManager.java | 22 ------------------- 4 files changed, 2 insertions(+), 37 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 69c9034c15..cb409ad7c7 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -167,7 +167,7 @@ public class Case implements SleuthkitCase.ErrorObserver { * * @throws IllegalStateException if there is no case open. */ - public static Case getCurrentCase() throws IllegalStateException { + public static Case getCurrentCase() { if (currentCase != null) { return currentCase; } else { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 42f518b5e4..b0cb987710 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -470,8 +470,7 @@ public class IngestManager { } /** - * Queries whether or not any ingest jobs are in progress, at least at the - * moment of the query. + * Queries whether or not any ingest jobs are in progress. * * @return True or false. */ diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java index a6ba0e5483..84d1434644 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java @@ -69,18 +69,6 @@ public class TikaFileTypeDetector { buf = buffer; } - // the xml detection in Tika tries to parse the entire file and throws exceptions - // for files that are not valid XML - try { - String tagHeader = new String(buf, 0, 5); - if (tagHeader.equals(" Date: Tue, 23 Dec 2014 10:01:16 -0500 Subject: [PATCH 78/84] Rename UserDefinedFileTypeIdentifier --- .../modules/filetypeid/FileTypeIdIngestModule.java | 4 ++-- ...peIdentifier.java => UserDefinedFileTypeDetector.java} | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) rename Core/src/org/sleuthkit/autopsy/modules/filetypeid/{UserDefinedFileTypeIdentifier.java => UserDefinedFileTypeDetector.java} (89%) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index c3c6bbcb5b..9713efd899 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -49,7 +49,7 @@ public class FileTypeIdIngestModule implements FileIngestModule { private long jobId; private static final HashMap totalsForIngestJobs = new HashMap<>(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); - private final UserDefinedFileTypeIdentifier userDefinedFileTypeIdentifier; + private final UserDefinedFileTypeDetector userDefinedFileTypeIdentifier; private final TikaFileTypeDetector tikaDetector = new TikaFileTypeDetector(); /** @@ -77,7 +77,7 @@ public class FileTypeIdIngestModule implements FileIngestModule { */ FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { this.settings = settings; - userDefinedFileTypeIdentifier = new UserDefinedFileTypeIdentifier(); + userDefinedFileTypeIdentifier = new UserDefinedFileTypeDetector(); try { userDefinedFileTypeIdentifier.loadFileTypes(); } catch (UserDefinedFileTypesManager.UserDefinedFileTypesException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java similarity index 89% rename from Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java index f756785585..bdbeaa3b72 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeIdentifier.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/UserDefinedFileTypeDetector.java @@ -23,17 +23,17 @@ import java.util.Map; import org.sleuthkit.datamodel.AbstractFile; /** - * Does file type identification for user-defined file types. + * Does file type detection for user-defined file types. */ -final class UserDefinedFileTypeIdentifier { +final class UserDefinedFileTypeDetector { private Map fileTypes; /** - * Creates an object that can do file type identification for user-defined + * Creates an object that can do file type detection for user-defined * file types. */ - UserDefinedFileTypeIdentifier() { + UserDefinedFileTypeDetector() { fileTypes = new HashMap<>(); } From 1ce585e9061d6ac2c2e6b7fa93255f4314164b37 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Dec 2014 10:25:28 -0500 Subject: [PATCH 79/84] Add missing synchronized keyword to IngestJob method --- Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index fe129c3528..8218186b4c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -73,7 +73,7 @@ public final class IngestJob { * * @return True or false. */ - boolean hasIngestPipeline() { + synchronized boolean hasIngestPipeline() { for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { if (dataSourceJob.hasIngestPipeline()) { return true; From e21acc818cacf4401bcc09d8bc2b66938fe04f71 Mon Sep 17 00:00:00 2001 From: Karl Mortensen Date: Fri, 2 Jan 2015 11:51:26 -0500 Subject: [PATCH 80/84] close file handles appropriately --- .../PhotoRecCarverFileIngestModule.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index 7accfe711b..cd5723e377 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -175,7 +175,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { processAndSettings.redirectOutput(Redirect.appendTo(log)); int exitValue = ExecUtil.execute(processAndSettings, new FileIngestModuleProcessTerminator(this.context)); - + if (this.context.fileIngestIsCancelled() == true) { // if it was cancelled by the user, result is OK // cleanup the output path @@ -205,13 +205,14 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { oldAuditFile.renameTo(newAuditFile); Path pathToRemove = Paths.get(outputDirPath.toAbsolutePath().toString()); - DirectoryStream stream = Files.newDirectoryStream(pathToRemove); - for (Path entry : stream) { - if (Files.isDirectory(entry)) { - FileUtil.deleteDir(new File(entry.toString())); + try (DirectoryStream stream = Files.newDirectoryStream(pathToRemove)) { + for (Path entry : stream) { + if (Files.isDirectory(entry)) { + FileUtil.deleteDir(new File(entry.toString())); + } } } - + // Now that we've cleaned up the folders and data files, parse the xml output file to add carved items into the database PhotoRecCarverOutputParser parser = new PhotoRecCarverOutputParser(outputDirPath); List theList = parser.parse(newAuditFile, id, file); From 40731f78b368dcf5e10e8c16a2307abf2a6949a8 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 8 Jan 2015 23:03:56 -0500 Subject: [PATCH 81/84] Expanded on making report panel more --- docs/doxygen/modReport.dox | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/doxygen/modReport.dox b/docs/doxygen/modReport.dox index c61471f5f1..17d7006fff 100644 --- a/docs/doxygen/modReport.dox +++ b/docs/doxygen/modReport.dox @@ -54,6 +54,7 @@ If you implement FileReportModule, the overriden methods will be: As when generating table module reports, Autopsy will iterate through a list of user selected data (which are represented by FileReportDataTypes), and call addRow(AbstractFile toAdd, List columns) for every abstract file in the case. Developers are guaranteed that the order of method calls will be startReport(), startTable(List headers), addRow(AbstractFile toAdd, List columns), AbstractFile toAdd, List columns),..., endTable(), endReport(). \subsection report_create_module_general Creating a General Report Module + If you implement GeneralReportModule, the overriden methods will be: - org.sleuthkit.autopsy.report.GeneralReportModule.generateReport(String reportPath, ReportProgressPanel progressPanel) - org.sleuthkit.autopsy.report.GeneralReportModule.getConfigurationPanel() @@ -62,7 +63,8 @@ For general report modules, Autopsy will simply call the generateReport(String r When updating the progress panel, it is recommended to update it as infrequently as possible, while still keeping the user informed. If your report processes 100,000 files and you chose to update the UI each time a file is reviewed, the UI would freeze when trying to process all your requests. This would cause problems to not only your reporting module, but to other modules running in parallel. A safer approach would be to update the UI every 1,000 files, or when a certain "category" of the files being processed has changed. For example, the HTML report module increments the progress bar and changes the processing label every time a new Blackboard Artifact Type is being processed. -Autopsy will also display the panel returned by getConfigurationPanel() in the generate report wizard. This panel can be used to allow the user custom controls over the report. +Autopsy will also display the panel returned by getConfigurationPanel() in the generate report wizard. This panel can be used to allow the user custom controls over the report. To make this panel, use NetBeans to make a new JPanel class and use its layout interface to put the UI widgets in the places that you want them. Then, your getConfigurationPanel() method should create an instance of that class and return it. + Typically a general report module should interact with both the Blackboard API in the org.sleuthkit.datamodel.SleuthkitCase class, in addition to an API (possibly external/thirdparty) to convert Blackboard Artifacts to the desired reporting format. From ac1b6a8d8c7bac16e4a08e6a2eb2c9d0cd4deec2 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 13 Jan 2015 11:47:37 -0500 Subject: [PATCH 82/84] Supply missing position attribute in top level registration of FileTypeIdOptionsPanelController --- .../modules/filetypeid/FileTypeIdOptionsPanelController.java | 3 ++- .../InterestingItemDefsOptionsPanelController.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java index 8f9aa7b500..a78abb0c5c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdOptionsPanelController.java @@ -16,7 +16,8 @@ import org.openide.util.Lookup; categoryName = "#OptionsCategory_Name_FileTypeId", iconBase = "org/sleuthkit/autopsy/modules/filetypeid/user-defined-file-types-settings.png", keywords = "#OptionsCategory_Keywords_FileTypeId", - keywordsCategory = "FileTypeId" + keywordsCategory = "FileTypeId", + position = 5 ) @org.openide.util.NbBundle.Messages({"OptionsCategory_Name_FileTypeId=FileTypeId", "OptionsCategory_Keywords_FileTypeId=FileTypeId"}) public final class FileTypeIdOptionsPanelController extends OptionsPanelController { diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java index c845bb71e0..b23ecb8dc0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsOptionsPanelController.java @@ -31,7 +31,7 @@ import org.openide.util.Lookup; iconBase = "org/sleuthkit/autopsy/images/interesting_item_32x32.png", keywords = "#OptionsCategory_Keywords_InterestingItemDefinitions", keywordsCategory = "InterestingItemDefinitions", - position = 5 + position = 6 ) public final class InterestingItemDefsOptionsPanelController extends OptionsPanelController { From 34bbdb1f57268ec90ebcdf6940de00a7712b8b07 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 14 Jan 2015 10:18:43 -0500 Subject: [PATCH 83/84] Changed file extraction so that it replaces : with _ so that attributes are not hidden as ADS --- .../autopsy/directorytree/ExtractAction.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java index 72681fba2a..fd2fce803a 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java @@ -91,17 +91,28 @@ public final class ExtractAction extends AbstractAction { } } - private void extractFile(ActionEvent e, AbstractFile source) { + /** + * Called when user has selected a single file to extract + * @param e + * @param selectedFile Selected file + */ + private void extractFile(ActionEvent e, AbstractFile selectedFile) { JFileChooser fileChooser = new JFileChooser(); fileChooser.setCurrentDirectory(new File(Case.getCurrentCase().getExportDirectory())); - fileChooser.setSelectedFile(new File(source.getName())); + // If there is an attribute name, change the ":". Otherwise the extracted file will be hidden + fileChooser.setSelectedFile(new File(selectedFile.getName().replace(':', '_'))); if (fileChooser.showSaveDialog((Component)e.getSource()) == JFileChooser.APPROVE_OPTION) { ArrayList fileExtractionTasks = new ArrayList<>(); - fileExtractionTasks.add(new FileExtractionTask(source, fileChooser.getSelectedFile())); - doFileExtraction(e, fileExtractionTasks); + fileExtractionTasks.add(new FileExtractionTask(selectedFile, fileChooser.getSelectedFile())); + runExtractionTasks(e, fileExtractionTasks); } } + /** + * Called when a user has selected multiple files to extract + * @param e + * @param selectedFiles Selected files + */ private void extractFiles(ActionEvent e, Collection selectedFiles) { JFileChooser folderChooser = new JFileChooser(); folderChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); @@ -120,15 +131,17 @@ public final class ExtractAction extends AbstractAction { } } + // make a task for each file ArrayList fileExtractionTasks = new ArrayList<>(); for (AbstractFile source : selectedFiles) { - fileExtractionTasks.add(new FileExtractionTask(source, new File(destinationFolder, source.getId() + "-" + source.getName()))); + // If there is an attribute name, change the ":". Otherwise the extracted file will be hidden + fileExtractionTasks.add(new FileExtractionTask(source, new File(destinationFolder, source.getId() + "-" + source.getName().replace(':', '_')))); } - doFileExtraction(e, fileExtractionTasks); + runExtractionTasks(e, fileExtractionTasks); } } - private void doFileExtraction(ActionEvent e, ArrayList fileExtractionTasks) { + private void runExtractionTasks(ActionEvent e, ArrayList fileExtractionTasks) { // verify all of the sources and destinations are OK for (Iterator it = fileExtractionTasks.iterator(); it.hasNext(); ) { @@ -162,6 +175,7 @@ public final class ExtractAction extends AbstractAction { } } + // launch a thread to do the work if (!fileExtractionTasks.isEmpty()) { try { FileExtracter extracter = new FileExtracter(fileExtractionTasks); @@ -187,6 +201,9 @@ public final class ExtractAction extends AbstractAction { } } + /** + * Thread that does the actual extraction work + */ private class FileExtracter extends SwingWorker { private Logger logger = Logger.getLogger(FileExtracter.class.getName()); private ProgressHandle progress; From 25258717df521e2800544e3f903227e73426265e Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 14 Jan 2015 12:38:07 -0500 Subject: [PATCH 84/84] Added comments to why dataresultviewertable drops first property. Made ImageNode name be more useful --- .../corecomponents/DataResultViewerTable.java | 26 +++++++++++-------- .../autopsy/datamodel/ImageNode.java | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index d6ad2e9bda..a5abcb1c23 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -307,26 +307,30 @@ public class DataResultViewerTable extends AbstractDataResultViewer { DataResultViewerTable.this.getAllChildPropertyHeadersRec(root, 100); List> props = new ArrayList<>(propertiesAcc); + + /* OutlineView makes the first column be the result of node.getDisplayName with the icon. This + * duplicates our first column, which is the file name, etc. So, pop that property off the list, but + * use its display name as the header for the column so that the header can change depending on the + * type of data being displayed. + * + * NOTE: This assumes that the first property is always the one tha duplicates getDisplayName(). This + * seems like a big assumption and could be made more robust. + */ if (props.size() > 0) { Node.Property prop = props.remove(0); ((DefaultOutlineModel) ov.getOutline().getOutlineModel()).setNodesColumnLabel(prop.getDisplayName()); } - // *********** Make the TreeTableView to be sortable *************** - - //First property column is sortable, but also sorted initially, so - //initially this one will have the arrow icon: - if (props.size() > 0) { - props.get(0).setValue("TreeColumnTTV", Boolean.TRUE); // Identifies special property representing first (tree) column. NON-NLS - props.get(0).setValue("SortingColumnTTV", Boolean.TRUE); // TreeTableView should be initially sorted by this property column. NON-NLS - } - - // The rest of the columns are sortable, but not initially sorted, - // so initially will have no arrow icon: + // Get the columns setup with respect to names and sortability String[] propStrings = new String[props.size() * 2]; for (int i = 0; i < props.size(); i++) { props.get(i).setValue("ComparableColumnTTV", Boolean.TRUE); //NON-NLS + //First property column is sorted initially + if (i == 0) { + props.get(i).setValue("TreeColumnTTV", Boolean.TRUE); // Identifies special property representing first (tree) column. NON-NLS + props.get(i).setValue("SortingColumnTTV", Boolean.TRUE); // TreeTableView should be initially sorted by this property column. NON-NLS + } propStrings[2 * i] = props.get(i).getName(); propStrings[2 * i + 1] = props.get(i).getDisplayName(); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java index babc47cde8..3cc523b4dc 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java @@ -88,7 +88,7 @@ public class ImageNode extends AbstractContentNode { ss.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "ImageNode.createSheet.name.name"), NbBundle.getMessage(this.getClass(), "ImageNode.createSheet.name.displayName"), NbBundle.getMessage(this.getClass(), "ImageNode.createSheet.name.desc"), - getName())); + getDisplayName())); return s; }