From 1ae429aedef8b65441d086852285cf2b7b0d8ebd Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 1 Jun 2020 12:00:34 -0400 Subject: [PATCH] updates based on design doc --- .../modules/interestingitems/FilesSet.java | 42 ++- .../InterestingItemsFilesSetSettings.java | 27 +- .../StandardInterestingFileSetsLoader.java | 151 ---------- .../StandardInterestingFilesSetsLoader.java | 285 ++++++++++++++++++ 4 files changed, 347 insertions(+), 158 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFileSetsLoader.java create mode 100644 Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFilesSetsLoader.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java index 5c5c79163a..2ca2893ac8 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java @@ -44,7 +44,10 @@ public final class FilesSet implements Serializable { private final String description; private final boolean ignoreKnownFiles; private final boolean ignoreUnallocatedSpace; - private transient boolean readOnly = false; + + private final boolean readOnly; + private final int versionNumber; + private final Map rules = new HashMap<>(); /** @@ -60,9 +63,36 @@ public final class FilesSet implements Serializable { * but a set with no rules is the empty set. */ public FilesSet(String name, String description, boolean ignoreKnownFiles, boolean ignoreUnallocatedSpace, Map rules) { + this(name, description, ignoreKnownFiles, ignoreUnallocatedSpace, rules, false, 0); + } + + /** + * Constructs an interesting files set. + * + * @param name The name of the set. + * @param description A description of the set, may be null. + * @param ignoreKnownFiles Whether or not to exclude known files from + * the set. + * @param ignoreUnallocatedSpace Whether or not to exclude unallocated space + * from the set. + * @param readOnly Whether or not the FilesSet should be read only (if not it is editable). + * @param versionNumber The versionNumber for the FilesSet so that older versions can be replaced with newer versions. + * @param rules The rules that define the set. May be null, + * but a set with no rules is the empty set. + */ + public FilesSet(String name, String description, boolean ignoreKnownFiles, boolean ignoreUnallocatedSpace, Map rules, + boolean readOnly, int versionNumber) { if ((name == null) || (name.isEmpty())) { throw new IllegalArgumentException("Interesting files set name cannot be null or empty"); } + + if (versionNumber < 0) { + throw new IllegalArgumentException("version number must be >= 0"); + } + + this.readOnly = readOnly; + this.versionNumber = versionNumber; + this.name = name; this.description = (description != null ? description : ""); this.ignoreKnownFiles = ignoreKnownFiles; @@ -81,13 +111,13 @@ public final class FilesSet implements Serializable { } /** - *Sets whether or not the file set is read only. This is a transient field that is not - * @param readOnly Whether or not the file set should be read only. + * Returns he versionNumber for the FilesSet so that older versions can be replaced with newer versions. + * @return The versionNumber for the FilesSet so that older versions can be replaced with newer versions. */ - void setReadOnly(boolean readOnly) { - this.readOnly = readOnly; + int getVersionNumber() { + return versionNumber; } - + /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemsFilesSetSettings.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemsFilesSetSettings.java index f491fe14d6..c63b6b40cc 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemsFilesSetSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemsFilesSetSettings.java @@ -35,6 +35,7 @@ import java.util.regex.PatternSyntaxException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.commons.lang.StringUtils; import org.openide.util.io.NbObjectInputStream; import org.openide.util.io.NbObjectOutputStream; import org.sleuthkit.autopsy.coreutils.Logger; @@ -79,6 +80,8 @@ class InterestingItemsFilesSetSettings implements Serializable { private static final Logger logger = Logger.getLogger(InterestingItemsFilesSetSettings.class.getName()); private static final String TYPE_FILTER_ATTR = "typeFilter"; //NON-NLS private static final String EXTENSION_RULE_TAG = "EXTENSION"; //NON-NLS + private static final String READONLY = "readOnly"; + private static final String VERSION_NUMBER = "versionNumber"; private Map filesSets; @@ -378,6 +381,26 @@ class InterestingItemsFilesSetSettings implements Serializable { if (!ignoreUnallocated.isEmpty()) { ignoreUnallocatedSpace = Boolean.parseBoolean(ignoreUnallocated); } + + String isReadonlyString = setElem.getAttribute(READONLY); + boolean isReadOnly = false; + if (StringUtils.isNotBlank(isReadonlyString)) { + isReadOnly = Boolean.parseBoolean(isReadonlyString); + } + + String versionNumberString = setElem.getAttribute(VERSION_NUMBER); + int versionNumber = 0; + if (StringUtils.isNotBlank(isReadonlyString)) { + try { + versionNumber = Integer.parseInt(versionNumberString); + } + catch (NumberFormatException ex) { + logger.log(Level.WARNING, + String.format("Unable to parse version number for files set named: %s with provided input: '%s'", setName, versionNumberString), + ex); + } + } + // Read the set membership rules, if any. Map rules = new HashMap<>(); NodeList allRuleElems = setElem.getChildNodes(); @@ -401,7 +424,7 @@ class InterestingItemsFilesSetSettings implements Serializable { // Make the files set. Note that degenerate sets with no rules are // allowed to facilitate the separation of set definition and rule // definitions. A set without rules is simply the empty set. - FilesSet set = new FilesSet(setName, description, ignoreKnownFiles, ignoreUnallocatedSpace, rules); + FilesSet set = new FilesSet(setName, description, ignoreKnownFiles, ignoreUnallocatedSpace, rules, isReadOnly, versionNumber); filesSets.put(set.getName(), set); } // Note: This method takes a file path to support the possibility of @@ -518,6 +541,8 @@ class InterestingItemsFilesSetSettings implements Serializable { setElement.setAttribute(NAME_ATTR, set.getName()); setElement.setAttribute(DESC_ATTR, set.getDescription()); setElement.setAttribute(IGNORE_KNOWN_FILES_ATTR, Boolean.toString(set.ignoresKnownFiles())); + setElement.setAttribute(READONLY, Boolean.toString(set.isReadOnly())); + setElement.setAttribute(VERSION_NUMBER, Integer.toString(set.getVersionNumber())); // Add the child elements for the set membership rules. // All conditions of a rule will be written as a single element in the xml for (FilesSet.Rule rule : set.getRules().values()) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFileSetsLoader.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFileSetsLoader.java deleted file mode 100644 index 41f3bf969e..0000000000 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFileSetsLoader.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2020 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.modules.interestingitems; - -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; -import org.apache.commons.io.FileUtils; -import org.openide.modules.OnStart; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; - -/** - * When the interesting items module loads, this runnable loads standard - * interesting file set rules. - */ -@OnStart -public class StandardInterestingFileSetsLoader implements Runnable { - - private static final Logger LOGGER = Logger.getLogger(StandardInterestingFileSetsLoader.class.getName()); - - private static final String CONFIG_DIR = "InterestingFileSetRules"; - private static final FilenameFilter DEFAULT_XML_FILTER = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".xml"); - } - - }; - - @Override - public void run() { - File rulesConfigDir = new File(PlatformUtil.getUserConfigDirectory(), CONFIG_DIR); - - copyRulesDirectory(rulesConfigDir); - - Map standardInterestingFileSets = readStandardFileXML(rulesConfigDir); - - Map userConfiguredSettings = null; - try { - userConfiguredSettings = FilesSetsManager.getInstance().getInterestingFilesSets(); - } catch (FilesSetsManager.FilesSetsManagerException ex) { - LOGGER.log(Level.SEVERE, "Unable to properly read user-configured interesting files sets.", ex); - } - - if (userConfiguredSettings == null) { - return; - } - - // TODO the rest of this - - // Call InterestingItemsFilesSetSettings.readDefinitionsXML for each file in the InterestingFileSetRules directory, - // setting the read only flag of each (actually one) FilesSet in the returned Map objects and adding - // the Maps objects to a local Map object. - - - - //Call FilesSetManager.getInterestingFilesSets and add the Map to the local Map from step “b.” -//The ordering of “b” and “c” avoids overwriting any file set rules defined by the user that incidentally have the same rule set name as the standard rule set. -//Call FilesSetManager.setInterestingFilesSets with the Map from step “c.” - - } - - /** - * Reads xml definitions for each file found in the standard interesting file set config directory and marks the files set as readonly. - * @param rulesConfigDir The user configuration directory for standard interesting file set rules. This is assumed to be non-null. - * @return The mapping of files set keys to the file sets. - */ - private static Map readStandardFileXML(File rulesConfigDir) { - Map standardInterestingFileSets = new HashMap<>(); - if (rulesConfigDir.exists()) { - for (File standardFileSetsFile : rulesConfigDir.listFiles(DEFAULT_XML_FILTER)) { - try { - Map thisFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(standardFileSetsFile); - thisFilesSet.values().stream().forEach(filesSet -> filesSet.setReadOnly(true)); - - standardInterestingFileSets.putAll(thisFilesSet); - } catch (FilesSetsManager.FilesSetsManagerException ex) { - LOGGER.log(Level.WARNING, String.format("There was a problem importing the standard interesting file set at: %s.", - standardFileSetsFile.getAbsoluteFile()), ex); - } - } - } - return standardInterestingFileSets; - } - - /** - * Add the InterestingFileSetRules directory to the user’s app data config directory for Autopsy if not already present. - * @param rulesConfigDir The user configuration directory for standard interesting file set rules. This is assumed to be non-null. - */ - private static void copyRulesDirectory(File rulesConfigDir) { - if (rulesConfigDir.exists()) { - LOGGER.info(String.format("%s settings directory already exists. Not going to perform copy of class resource standard interesting files to directory.", - rulesConfigDir.getAbsolutePath())); - } - - rulesConfigDir.mkdirs(); - - if (!rulesConfigDir.exists()) { - LOGGER.severe( - String.format("Unable to create directory at %s. Failed to copy standard interesting file set rules to this directory.", - rulesConfigDir.getAbsolutePath())); - return; - } - - // taken from https://stackoverflow.com/a/19459180 - URL url = StandardInterestingFileSetsLoader.class.getClassLoader().getResource(CONFIG_DIR); - File resourceDirectory = null; - try { - resourceDirectory = new File(url.toURI()); - } catch (URISyntaxException ignored) { - resourceDirectory = new File(url.getPath()); - } - - if (resourceDirectory == null || !resourceDirectory.exists()) { - LOGGER.severe( - String.format("Unable to find resource directory for standard interesting file sets, %s.", - (rulesConfigDir != null) ? rulesConfigDir.getAbsolutePath() : "")); - return; - } - - try { - FileUtils.copyDirectory(resourceDirectory, rulesConfigDir); - } catch (IOException ex) { - LOGGER.log(Level.SEVERE, String.format("There was an error copying %s to %s.", - resourceDirectory.getAbsolutePath(), rulesConfigDir.getAbsolutePath()), ex); - } - } - -} diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFilesSetsLoader.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFilesSetsLoader.java new file mode 100644 index 0000000000..9c596d24f2 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/StandardInterestingFilesSetsLoader.java @@ -0,0 +1,285 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.interestingitems; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import org.apache.commons.io.FileUtils; +import org.openide.modules.OnStart; +import org.openide.util.NbBundle.Messages; +import org.openide.util.io.NbObjectOutputStream; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; + +/** + * When the interesting items module loads, this runnable loads standard + * interesting file set rules. + */ +@OnStart +public class StandardInterestingFilesSetsLoader implements Runnable { + + private static final Logger LOGGER = Logger.getLogger(StandardInterestingFilesSetsLoader.class.getName()); + + private static final String CONFIG_DIR = "InterestingFileSetRules"; + + private static final FilenameFilter DEFAULT_XML_FILTER = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".xml"); + } + }; + + @Override + public void run() { + File rulesConfigDir = new File(PlatformUtil.getUserConfigDirectory(), CONFIG_DIR); + + copyRulesDirectory(rulesConfigDir); + + Map standardInterestingFileSets = readStandardFileXML(rulesConfigDir); + + // Call FilesSetManager.getInterestingFilesSets() to get a Map of the existing rule sets. + Map userConfiguredSettings = null; + try { + userConfiguredSettings = FilesSetsManager.getInstance().getInterestingFilesSets(); + } catch (FilesSetsManager.FilesSetsManagerException ex) { + LOGGER.log(Level.SEVERE, "Unable to properly read user-configured interesting files sets.", ex); + } + + if (userConfiguredSettings == null) { + return; + } + + // Add each FilesSet read from the standard rules set XML files that is missing from the Map to the Map. + copyOnNewer(standardInterestingFileSets, userConfiguredSettings, true); + + try { + // Call FilesSetManager.setInterestingFilesSets with the updated Map. + FilesSetsManager.getInstance().setInterestingFilesSets(userConfiguredSettings); + } catch (FilesSetsManager.FilesSetsManagerException ex) { + LOGGER.log(Level.SEVERE, "Unable to write updated configuration for interesting files sets to config directory.", ex); + } + } + + /** + * Reads xml definitions for each file found in the standard interesting + * file set config directory and marks the files set as readonly. + * + * @param rulesConfigDir The user configuration directory for standard + * interesting file set rules. This is assumed to be + * non-null. + * + * @return The mapping of files set keys to the file sets. + */ + private static Map readStandardFileXML(File rulesConfigDir) { + Map standardInterestingFileSets = new HashMap<>(); + if (rulesConfigDir.exists()) { + for (File standardFileSetsFile : rulesConfigDir.listFiles(DEFAULT_XML_FILTER)) { + try { + Map thisFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(standardFileSetsFile); + copyOnNewer(standardInterestingFileSets, thisFilesSet); + } catch (FilesSetsManager.FilesSetsManagerException ex) { + LOGGER.log(Level.WARNING, String.format("There was a problem importing the standard interesting file set at: %s.", + standardFileSetsFile.getAbsoluteFile()), ex); + } + } + } + return standardInterestingFileSets; + } + + /** + * Add the InterestingFileSetRules directory to the user’s app data config + * directory for Autopsy if not already present. + * + * @param rulesConfigDir The user configuration directory for standard + * interesting file set rules. This is assumed to be + * non-null. + */ + private static void copyRulesDirectory(File rulesConfigDir) { + if (rulesConfigDir.exists()) { + LOGGER.info(String.format("%s settings directory already exists. Not going to perform copy of class resource standard interesting files to directory.", + rulesConfigDir.getAbsolutePath())); + } + + // taken from https://stackoverflow.com/a/19459180 + URL url = StandardInterestingFilesSetsLoader.class.getClassLoader().getResource(CONFIG_DIR); + File resourceDirectory = null; + try { + resourceDirectory = new File(url.toURI()); + } catch (URISyntaxException ignored) { + resourceDirectory = new File(url.getPath()); + } + + if (resourceDirectory == null || !resourceDirectory.exists()) { + LOGGER.severe( + String.format("Unable to find resource directory for standard interesting file sets, %s.", + (rulesConfigDir != null) ? rulesConfigDir.getAbsolutePath() : "")); + return; + } + + try { + for (File resourceFile : resourceDirectory.listFiles(DEFAULT_XML_FILTER)) { + updateStandardFilesSetConfigFile(rulesConfigDir, resourceFile); + } + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, String.format("There was an error copying %s to %s.", + resourceDirectory.getAbsolutePath(), rulesConfigDir.getAbsolutePath()), ex); + } + } + + /** + * Updates the standard interesting files set config file if there is no + * corresponding files set on disk or the files set on disk has an older + * version. + * + * @param rulesConfigDir The directory for standard interesting files sets. + * @param resourceFile The standard interesting files set resource file + * located within the jar. + * + * @throws IOException + */ + private static void updateStandardFilesSetConfigFile(File rulesConfigDir, File resourceFile) throws IOException { + File configDirFile = new File(rulesConfigDir, resourceFile.getName()); + + if (configDirFile.exists()) { + Map resourceFilesSet = null; + try { + resourceFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(resourceFile); + } catch (FilesSetsManager.FilesSetsManagerException ex) { + LOGGER.log(Level.SEVERE, "Unable to read FilesSet data from resource file: " + resourceFile.getName(), ex); + } + + Map configDirFilesSet = null; + try { + configDirFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(configDirFile); + } catch (FilesSetsManager.FilesSetsManagerException ex) { + LOGGER.log(Level.WARNING, "Unable to read FilesSet data from config file: " + resourceFile.getName(), ex); + } + + if (resourceFilesSet == null && configDirFilesSet != null) { + return; + } else if (configDirFilesSet != null && resourceFilesSet != null) { + Map newMapping = new HashMap<>(); + copyOnNewer(resourceFilesSet, newMapping); + copyOnNewer(configDirFilesSet, newMapping); + + try (final NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(configDirFile))) { + out.writeObject(new InterestingItemsFilesSetSettings(newMapping)); + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Unable to create new standard interesting files set for " + configDirFile.getPath(), ex); + } + } + } + + FileUtils.copyFileToDirectory(resourceFile, rulesConfigDir); + } + + /** + * Copies the entries in the src map to the destination map if the src item + * has a newer version than what is in dest or no equivalent entry exists + * within the dest map. + * + * @param src The source map. + * @param dest The destination map. + */ + private static void copyOnNewer(Map src, Map dest) { + copyOnNewer(src, dest, false); + } + + /** + * Copies the entries in the src map to the destination map if the src item + * has a newer version than what is in dest or no equivalent entry exists + * within the dest map. + * + * @param src The source map. + * @param dest The destination map. + * @param appendCustom On conflict, if one of the items is readonly and one + * is not, this flag can be set so the item that is not + * readonly will have " (custom)" appended. + */ + private static void copyOnNewer(Map src, Map dest, boolean appendCustom) { + for (Map.Entry srcEntry : src.entrySet()) { + String key = srcEntry.getKey(); + FilesSet srcFileSet = srcEntry.getValue(); + FilesSet destFileSet = dest.get(key); + if (destFileSet != null) { + // If and only if there is a naming conflict with a user-defined rule set, append “(Custom)” + // to the user-defined rule set and add it back to the Map. + if (appendCustom && srcFileSet.isReadOnly() != destFileSet.isReadOnly()) { + if (srcFileSet.isReadOnly()) { + addCustomFile(dest, key, destFileSet); + } else { + addCustomFile(dest, key, srcFileSet); + src.put(key, destFileSet); + } + continue; + } + + // Replace each FilesSet read from the standard rules set XML files that has a newer version + // number than the corresponding FilesSet in the Map with the updated FilesSet. + if (destFileSet.getVersionNumber() >= srcEntry.getValue().getVersionNumber()) { + continue; + } + } + + dest.put(srcEntry.getKey(), srcEntry.getValue()); + } + } + + /** + * Adds an entry to the destination map where the name will be the same as + * the key with " (custom)" appended. + * + * @param dest The destination map. + * @param key The key that will be used for the basis of the name + * and the key in the hashmap ("custom" will be + * appended). + * @param srcFilesSet The FilesSet to append as custom. A non-readonly + * filesset must be provided. + */ + @Messages({ + "# {0} - filesSetName", + "StandardInterestingFileSetsLoader.customSuffixed={0} (Custom)" + }) + private static void addCustomFile(Map dest, String key, FilesSet srcFilesSet) { + if (srcFilesSet.isReadOnly()) { + LOGGER.log(Level.SEVERE, "An attempt to create a custom file that was not readonly"); + return; + } + + String customKey = Bundle.StandardInterestingFileSetsLoader_customSuffixed(key); + FilesSet customFilesSet = new FilesSet( + customKey, + srcFilesSet.getDescription(), + srcFilesSet.ignoresKnownFiles(), + srcFilesSet.ingoresUnallocatedSpace(), + srcFilesSet.getRules(), + false, + srcFilesSet.getVersionNumber() + ); + dest.put(customKey, customFilesSet); + } + +}