updates based on design doc

This commit is contained in:
Greg DiCristofaro
2020-06-01 12:00:34 -04:00
parent 33b77c37f5
commit 1ae429aede
4 changed files with 347 additions and 158 deletions
@@ -44,7 +44,10 @@ public final class FilesSet implements Serializable {
private final String description;
private final boolean ignoreKnownFiles;
private final boolean ignoreUnallocatedSpace;
private transient boolean readOnly = false;
private final boolean readOnly;
private final int versionNumber;
private final Map<String, Rule> rules = new HashMap<>();
/**
@@ -60,9 +63,36 @@ public final class FilesSet implements Serializable {
* but a set with no rules is the empty set.
*/
public FilesSet(String name, String description, boolean ignoreKnownFiles, boolean ignoreUnallocatedSpace, Map<String, Rule> rules) {
this(name, description, ignoreKnownFiles, ignoreUnallocatedSpace, rules, false, 0);
}
/**
* Constructs an interesting files set.
*
* @param name The name of the set.
* @param description A description of the set, may be null.
* @param ignoreKnownFiles Whether or not to exclude known files from
* the set.
* @param ignoreUnallocatedSpace Whether or not to exclude unallocated space
* from the set.
* @param readOnly Whether or not the FilesSet should be read only (if not it is editable).
* @param versionNumber The versionNumber for the FilesSet so that older versions can be replaced with newer versions.
* @param rules The rules that define the set. May be null,
* but a set with no rules is the empty set.
*/
public FilesSet(String name, String description, boolean ignoreKnownFiles, boolean ignoreUnallocatedSpace, Map<String, Rule> rules,
boolean readOnly, int versionNumber) {
if ((name == null) || (name.isEmpty())) {
throw new IllegalArgumentException("Interesting files set name cannot be null or empty");
}
if (versionNumber < 0) {
throw new IllegalArgumentException("version number must be >= 0");
}
this.readOnly = readOnly;
this.versionNumber = versionNumber;
this.name = name;
this.description = (description != null ? description : "");
this.ignoreKnownFiles = ignoreKnownFiles;
@@ -81,13 +111,13 @@ public final class FilesSet implements Serializable {
}
/**
*Sets whether or not the file set is read only. This is a transient field that is not
* @param readOnly Whether or not the file set should be read only.
* Returns he versionNumber for the FilesSet so that older versions can be replaced with newer versions.
* @return The versionNumber for the FilesSet so that older versions can be replaced with newer versions.
*/
void setReadOnly(boolean readOnly) {
this.readOnly = readOnly;
int getVersionNumber() {
return versionNumber;
}
/**
@@ -35,6 +35,7 @@ import java.util.regex.PatternSyntaxException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.StringUtils;
import org.openide.util.io.NbObjectInputStream;
import org.openide.util.io.NbObjectOutputStream;
import org.sleuthkit.autopsy.coreutils.Logger;
@@ -79,6 +80,8 @@ class InterestingItemsFilesSetSettings implements Serializable {
private static final Logger logger = Logger.getLogger(InterestingItemsFilesSetSettings.class.getName());
private static final String TYPE_FILTER_ATTR = "typeFilter"; //NON-NLS
private static final String EXTENSION_RULE_TAG = "EXTENSION"; //NON-NLS
private static final String READONLY = "readOnly";
private static final String VERSION_NUMBER = "versionNumber";
private Map<String, FilesSet> filesSets;
@@ -378,6 +381,26 @@ class InterestingItemsFilesSetSettings implements Serializable {
if (!ignoreUnallocated.isEmpty()) {
ignoreUnallocatedSpace = Boolean.parseBoolean(ignoreUnallocated);
}
String isReadonlyString = setElem.getAttribute(READONLY);
boolean isReadOnly = false;
if (StringUtils.isNotBlank(isReadonlyString)) {
isReadOnly = Boolean.parseBoolean(isReadonlyString);
}
String versionNumberString = setElem.getAttribute(VERSION_NUMBER);
int versionNumber = 0;
if (StringUtils.isNotBlank(isReadonlyString)) {
try {
versionNumber = Integer.parseInt(versionNumberString);
}
catch (NumberFormatException ex) {
logger.log(Level.WARNING,
String.format("Unable to parse version number for files set named: %s with provided input: '%s'", setName, versionNumberString),
ex);
}
}
// Read the set membership rules, if any.
Map<String, FilesSet.Rule> rules = new HashMap<>();
NodeList allRuleElems = setElem.getChildNodes();
@@ -401,7 +424,7 @@ class InterestingItemsFilesSetSettings implements Serializable {
// Make the files set. Note that degenerate sets with no rules are
// allowed to facilitate the separation of set definition and rule
// definitions. A set without rules is simply the empty set.
FilesSet set = new FilesSet(setName, description, ignoreKnownFiles, ignoreUnallocatedSpace, rules);
FilesSet set = new FilesSet(setName, description, ignoreKnownFiles, ignoreUnallocatedSpace, rules, isReadOnly, versionNumber);
filesSets.put(set.getName(), set);
}
// Note: This method takes a file path to support the possibility of
@@ -518,6 +541,8 @@ class InterestingItemsFilesSetSettings implements Serializable {
setElement.setAttribute(NAME_ATTR, set.getName());
setElement.setAttribute(DESC_ATTR, set.getDescription());
setElement.setAttribute(IGNORE_KNOWN_FILES_ATTR, Boolean.toString(set.ignoresKnownFiles()));
setElement.setAttribute(READONLY, Boolean.toString(set.isReadOnly()));
setElement.setAttribute(VERSION_NUMBER, Integer.toString(set.getVersionNumber()));
// Add the child elements for the set membership rules.
// All conditions of a rule will be written as a single element in the xml
for (FilesSet.Rule rule : set.getRules().values()) {
@@ -1,151 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.modules.interestingitems;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import org.apache.commons.io.FileUtils;
import org.openide.modules.OnStart;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
/**
* When the interesting items module loads, this runnable loads standard
* interesting file set rules.
*/
@OnStart
public class StandardInterestingFileSetsLoader implements Runnable {
private static final Logger LOGGER = Logger.getLogger(StandardInterestingFileSetsLoader.class.getName());
private static final String CONFIG_DIR = "InterestingFileSetRules";
private static final FilenameFilter DEFAULT_XML_FILTER = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".xml");
}
};
@Override
public void run() {
File rulesConfigDir = new File(PlatformUtil.getUserConfigDirectory(), CONFIG_DIR);
copyRulesDirectory(rulesConfigDir);
Map<String, FilesSet> standardInterestingFileSets = readStandardFileXML(rulesConfigDir);
Map<String, FilesSet> userConfiguredSettings = null;
try {
userConfiguredSettings = FilesSetsManager.getInstance().getInterestingFilesSets();
} catch (FilesSetsManager.FilesSetsManagerException ex) {
LOGGER.log(Level.SEVERE, "Unable to properly read user-configured interesting files sets.", ex);
}
if (userConfiguredSettings == null) {
return;
}
// TODO the rest of this
// Call InterestingItemsFilesSetSettings.readDefinitionsXML for each file in the InterestingFileSetRules directory,
// setting the read only flag of each (actually one) FilesSet in the returned Map<String, FilesSet> objects and adding
// the Maps objects to a local Map<String, FilesSet> object.
//Call FilesSetManager.getInterestingFilesSets and add the Map<String, FilesSet> to the local Map<String, FilesSet> from step “b.”
//The ordering of “b” and “c” avoids overwriting any file set rules defined by the user that incidentally have the same rule set name as the standard rule set.
//Call FilesSetManager.setInterestingFilesSets with the Map<String, FilesSet> from step “c.”
}
/**
* Reads xml definitions for each file found in the standard interesting file set config directory and marks the files set as readonly.
* @param rulesConfigDir The user configuration directory for standard interesting file set rules. This is assumed to be non-null.
* @return The mapping of files set keys to the file sets.
*/
private static Map<String, FilesSet> readStandardFileXML(File rulesConfigDir) {
Map<String, FilesSet> standardInterestingFileSets = new HashMap<>();
if (rulesConfigDir.exists()) {
for (File standardFileSetsFile : rulesConfigDir.listFiles(DEFAULT_XML_FILTER)) {
try {
Map<String, FilesSet> thisFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(standardFileSetsFile);
thisFilesSet.values().stream().forEach(filesSet -> filesSet.setReadOnly(true));
standardInterestingFileSets.putAll(thisFilesSet);
} catch (FilesSetsManager.FilesSetsManagerException ex) {
LOGGER.log(Level.WARNING, String.format("There was a problem importing the standard interesting file set at: %s.",
standardFileSetsFile.getAbsoluteFile()), ex);
}
}
}
return standardInterestingFileSets;
}
/**
* Add the InterestingFileSetRules directory to the user’s app data config directory for Autopsy if not already present.
* @param rulesConfigDir The user configuration directory for standard interesting file set rules. This is assumed to be non-null.
*/
private static void copyRulesDirectory(File rulesConfigDir) {
if (rulesConfigDir.exists()) {
LOGGER.info(String.format("%s settings directory already exists. Not going to perform copy of class resource standard interesting files to directory.",
rulesConfigDir.getAbsolutePath()));
}
rulesConfigDir.mkdirs();
if (!rulesConfigDir.exists()) {
LOGGER.severe(
String.format("Unable to create directory at %s. Failed to copy standard interesting file set rules to this directory.",
rulesConfigDir.getAbsolutePath()));
return;
}
// taken from https://stackoverflow.com/a/19459180
URL url = StandardInterestingFileSetsLoader.class.getClassLoader().getResource(CONFIG_DIR);
File resourceDirectory = null;
try {
resourceDirectory = new File(url.toURI());
} catch (URISyntaxException ignored) {
resourceDirectory = new File(url.getPath());
}
if (resourceDirectory == null || !resourceDirectory.exists()) {
LOGGER.severe(
String.format("Unable to find resource directory for standard interesting file sets, %s.",
(rulesConfigDir != null) ? rulesConfigDir.getAbsolutePath() : "<null>"));
return;
}
try {
FileUtils.copyDirectory(resourceDirectory, rulesConfigDir);
} catch (IOException ex) {
LOGGER.log(Level.SEVERE, String.format("There was an error copying %s to %s.",
resourceDirectory.getAbsolutePath(), rulesConfigDir.getAbsolutePath()), ex);
}
}
}
@@ -0,0 +1,285 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.modules.interestingitems;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import org.apache.commons.io.FileUtils;
import org.openide.modules.OnStart;
import org.openide.util.NbBundle.Messages;
import org.openide.util.io.NbObjectOutputStream;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
/**
* When the interesting items module loads, this runnable loads standard
* interesting file set rules.
*/
@OnStart
public class StandardInterestingFilesSetsLoader implements Runnable {
private static final Logger LOGGER = Logger.getLogger(StandardInterestingFilesSetsLoader.class.getName());
private static final String CONFIG_DIR = "InterestingFileSetRules";
private static final FilenameFilter DEFAULT_XML_FILTER = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".xml");
}
};
@Override
public void run() {
File rulesConfigDir = new File(PlatformUtil.getUserConfigDirectory(), CONFIG_DIR);
copyRulesDirectory(rulesConfigDir);
Map<String, FilesSet> standardInterestingFileSets = readStandardFileXML(rulesConfigDir);
// Call FilesSetManager.getInterestingFilesSets() to get a Map<String, FilesSet> of the existing rule sets.
Map<String, FilesSet> userConfiguredSettings = null;
try {
userConfiguredSettings = FilesSetsManager.getInstance().getInterestingFilesSets();
} catch (FilesSetsManager.FilesSetsManagerException ex) {
LOGGER.log(Level.SEVERE, "Unable to properly read user-configured interesting files sets.", ex);
}
if (userConfiguredSettings == null) {
return;
}
// Add each FilesSet read from the standard rules set XML files that is missing from the Map to the Map.
copyOnNewer(standardInterestingFileSets, userConfiguredSettings, true);
try {
// Call FilesSetManager.setInterestingFilesSets with the updated Map.
FilesSetsManager.getInstance().setInterestingFilesSets(userConfiguredSettings);
} catch (FilesSetsManager.FilesSetsManagerException ex) {
LOGGER.log(Level.SEVERE, "Unable to write updated configuration for interesting files sets to config directory.", ex);
}
}
/**
* Reads xml definitions for each file found in the standard interesting
* file set config directory and marks the files set as readonly.
*
* @param rulesConfigDir The user configuration directory for standard
* interesting file set rules. This is assumed to be
* non-null.
*
* @return The mapping of files set keys to the file sets.
*/
private static Map<String, FilesSet> readStandardFileXML(File rulesConfigDir) {
Map<String, FilesSet> standardInterestingFileSets = new HashMap<>();
if (rulesConfigDir.exists()) {
for (File standardFileSetsFile : rulesConfigDir.listFiles(DEFAULT_XML_FILTER)) {
try {
Map<String, FilesSet> thisFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(standardFileSetsFile);
copyOnNewer(standardInterestingFileSets, thisFilesSet);
} catch (FilesSetsManager.FilesSetsManagerException ex) {
LOGGER.log(Level.WARNING, String.format("There was a problem importing the standard interesting file set at: %s.",
standardFileSetsFile.getAbsoluteFile()), ex);
}
}
}
return standardInterestingFileSets;
}
/**
* Add the InterestingFileSetRules directory to the user’s app data config
* directory for Autopsy if not already present.
*
* @param rulesConfigDir The user configuration directory for standard
* interesting file set rules. This is assumed to be
* non-null.
*/
private static void copyRulesDirectory(File rulesConfigDir) {
if (rulesConfigDir.exists()) {
LOGGER.info(String.format("%s settings directory already exists. Not going to perform copy of class resource standard interesting files to directory.",
rulesConfigDir.getAbsolutePath()));
}
// taken from https://stackoverflow.com/a/19459180
URL url = StandardInterestingFilesSetsLoader.class.getClassLoader().getResource(CONFIG_DIR);
File resourceDirectory = null;
try {
resourceDirectory = new File(url.toURI());
} catch (URISyntaxException ignored) {
resourceDirectory = new File(url.getPath());
}
if (resourceDirectory == null || !resourceDirectory.exists()) {
LOGGER.severe(
String.format("Unable to find resource directory for standard interesting file sets, %s.",
(rulesConfigDir != null) ? rulesConfigDir.getAbsolutePath() : "<null>"));
return;
}
try {
for (File resourceFile : resourceDirectory.listFiles(DEFAULT_XML_FILTER)) {
updateStandardFilesSetConfigFile(rulesConfigDir, resourceFile);
}
} catch (IOException ex) {
LOGGER.log(Level.SEVERE, String.format("There was an error copying %s to %s.",
resourceDirectory.getAbsolutePath(), rulesConfigDir.getAbsolutePath()), ex);
}
}
/**
* Updates the standard interesting files set config file if there is no
* corresponding files set on disk or the files set on disk has an older
* version.
*
* @param rulesConfigDir The directory for standard interesting files sets.
* @param resourceFile The standard interesting files set resource file
* located within the jar.
*
* @throws IOException
*/
private static void updateStandardFilesSetConfigFile(File rulesConfigDir, File resourceFile) throws IOException {
File configDirFile = new File(rulesConfigDir, resourceFile.getName());
if (configDirFile.exists()) {
Map<String, FilesSet> resourceFilesSet = null;
try {
resourceFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(resourceFile);
} catch (FilesSetsManager.FilesSetsManagerException ex) {
LOGGER.log(Level.SEVERE, "Unable to read FilesSet data from resource file: " + resourceFile.getName(), ex);
}
Map<String, FilesSet> configDirFilesSet = null;
try {
configDirFilesSet = InterestingItemsFilesSetSettings.readDefinitionsXML(configDirFile);
} catch (FilesSetsManager.FilesSetsManagerException ex) {
LOGGER.log(Level.WARNING, "Unable to read FilesSet data from config file: " + resourceFile.getName(), ex);
}
if (resourceFilesSet == null && configDirFilesSet != null) {
return;
} else if (configDirFilesSet != null && resourceFilesSet != null) {
Map<String, FilesSet> newMapping = new HashMap<>();
copyOnNewer(resourceFilesSet, newMapping);
copyOnNewer(configDirFilesSet, newMapping);
try (final NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(configDirFile))) {
out.writeObject(new InterestingItemsFilesSetSettings(newMapping));
} catch (IOException ex) {
LOGGER.log(Level.SEVERE, "Unable to create new standard interesting files set for " + configDirFile.getPath(), ex);
}
}
}
FileUtils.copyFileToDirectory(resourceFile, rulesConfigDir);
}
/**
* Copies the entries in the src map to the destination map if the src item
* has a newer version than what is in dest or no equivalent entry exists
* within the dest map.
*
* @param src The source map.
* @param dest The destination map.
*/
private static void copyOnNewer(Map<String, FilesSet> src, Map<String, FilesSet> dest) {
copyOnNewer(src, dest, false);
}
/**
* Copies the entries in the src map to the destination map if the src item
* has a newer version than what is in dest or no equivalent entry exists
* within the dest map.
*
* @param src The source map.
* @param dest The destination map.
* @param appendCustom On conflict, if one of the items is readonly and one
* is not, this flag can be set so the item that is not
* readonly will have " (custom)" appended.
*/
private static void copyOnNewer(Map<String, FilesSet> src, Map<String, FilesSet> dest, boolean appendCustom) {
for (Map.Entry<String, FilesSet> srcEntry : src.entrySet()) {
String key = srcEntry.getKey();
FilesSet srcFileSet = srcEntry.getValue();
FilesSet destFileSet = dest.get(key);
if (destFileSet != null) {
// If and only if there is a naming conflict with a user-defined rule set, append “(Custom)”
// to the user-defined rule set and add it back to the Map.
if (appendCustom && srcFileSet.isReadOnly() != destFileSet.isReadOnly()) {
if (srcFileSet.isReadOnly()) {
addCustomFile(dest, key, destFileSet);
} else {
addCustomFile(dest, key, srcFileSet);
src.put(key, destFileSet);
}
continue;
}
// Replace each FilesSet read from the standard rules set XML files that has a newer version
// number than the corresponding FilesSet in the Map with the updated FilesSet.
if (destFileSet.getVersionNumber() >= srcEntry.getValue().getVersionNumber()) {
continue;
}
}
dest.put(srcEntry.getKey(), srcEntry.getValue());
}
}
/**
* Adds an entry to the destination map where the name will be the same as
* the key with " (custom)" appended.
*
* @param dest The destination map.
* @param key The key that will be used for the basis of the name
* and the key in the hashmap ("custom" will be
* appended).
* @param srcFilesSet The FilesSet to append as custom. A non-readonly
* filesset must be provided.
*/
@Messages({
"# {0} - filesSetName",
"StandardInterestingFileSetsLoader.customSuffixed={0} (Custom)"
})
private static void addCustomFile(Map<String, FilesSet> dest, String key, FilesSet srcFilesSet) {
if (srcFilesSet.isReadOnly()) {
LOGGER.log(Level.SEVERE, "An attempt to create a custom file that was not readonly");
return;
}
String customKey = Bundle.StandardInterestingFileSetsLoader_customSuffixed(key);
FilesSet customFilesSet = new FilesSet(
customKey,
srcFilesSet.getDescription(),
srcFilesSet.ignoresKnownFiles(),
srcFilesSet.ingoresUnallocatedSpace(),
srcFilesSet.getRules(),
false,
srcFilesSet.getVersionNumber()
);
dest.put(customKey, customFilesSet);
}
}