codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-09-03 14:30:01 +00:00
Code Issues Releases Wiki Activity

Merge pull request #3987 from dgrove727/4066_PreventDuplicateHashSetHits

Browse Source 
4066 prevent duplicate hash set hits
This commit is contained in:
Richard Cordovano
2018-08-03 14:07:10 -04:00
committed by GitHub
parent 71fcb4a342 d01f254d6b
commit fa05d237d8
5 changed files with 108 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Core/src/org/sleuthkit/autopsy/casemodule/services/Blackboard.java
View File

@@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.casemodule.services;
import java.io.Closeable;
import java.io.IOException;
import org.openide.util.Lookup;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
@@ -38,7 +37,7 @@ import org.sleuthkit.datamodel.TskDataException;
public final class Blackboard implements Closeable {
private SleuthkitCase caseDb;
/**
* Constructs a representation of the blackboard, a place where artifacts
* and their attributes are posted.
@@ -80,8 +79,8 @@ public final class Blackboard implements Closeable {
*
* @return A type object representing the artifact type.
*
* @throws BlackboardBlackboardException If there is a problem getting or
* adding the artifact type.
* @throws BlackboardException If there is a problem getting or adding the
* artifact type.
*/
public synchronized BlackboardArtifact.Type getOrAddArtifactType(String typeName, String displayName) throws BlackboardException {
if (null == caseDb) {
@@ -110,8 +109,8 @@ public final class Blackboard implements Closeable {
*
* @return A type object representing the attribute type.
*
* @throws BlackboardBlackboardException If there is a problem getting or
* adding the attribute type.
* @throws BlackboardException If there is a problem getting or adding the
* attribute type.
*/
public synchronized BlackboardAttribute.Type getOrAddAttributeType(String typeName, BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE valueType, String displayName) throws BlackboardException {
if (null == caseDb) {
@@ -140,7 +139,6 @@ public final class Blackboard implements Closeable {
caseDb = null;
}
/**
* A blackboard exception.
*/

4
Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties
View File

@@ -87,8 +87,8 @@ HashDbImportDatabaseDialog.importHashDbErr=Import Hash Set Error
HashDbImportDatabaseDialog.mustSelectHashDbFilePathMsg=A hash set file path must be selected.
HashDbImportDatabaseDialog.hashDbDoesNotExistMsg=The selected hash set does not exist.
HashDbImportDatabaseDialog.errorMessage.failedToOpenHashDbMsg=Failed to open hash set at {0}.
HashDbIngestModule.moduleName=Hash Lookup
HashDbIngestModule.moduleDescription=Identifies known and notable files using supplied hash sets, such as a standard NSRL hash set.
HashLookupModuleFactory.moduleName.text=Hash Lookup
HashLookupModuleFactory.moduleDescription.text=Identifies known and notable files using supplied hash sets, such as a standard NSRL hash set.
HashDbIngestModule.fileReadErrorMsg=Read Error\: {0}
HashDbIngestModule.calcHashValueErr=Error encountered while calculating the hash value for {0}.
HashDbIngestModule.hashLookupErrorMsg=Hash Lookup Error\: {0}

5
Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle_ja.properties
View File

@@ -76,8 +76,8 @@ HashDbImportDatabaseDialog.importHashDbErr=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\
HashDbImportDatabaseDialog.mustSelectHashDbFilePathMsg=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u306e\u30d5\u30a1\u30a4\u30eb\u30d1\u30b9\u306e\u9078\u629e\u304c\u5fc5\u8981\u3067\u3059\u3002
HashDbImportDatabaseDialog.hashDbDoesNotExistMsg=\u9078\u629e\u3055\u308c\u305f\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u306f\u5b58\u5728\u3057\u307e\u305b\u3093\u3002
HashDbImportDatabaseDialog.errorMessage.failedToOpenHashDbMsg={0}\u3067\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u958b\u304f\u306e\u306b\u5931\u6557\u3057\u307e\u3057\u305f\u3002
HashDbIngestModule.moduleName=\u30cf\u30c3\u30b7\u30e5\u30eb\u30c3\u30af\u30a2\u30c3\u30d7
HashDbIngestModule.moduleDescription=\u6a19\u6e96\u306eNSRL\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u306a\u3069\u3001\u63d0\u4f9b\u3055\u308c\u305f\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u5229\u7528\u3057\u3066\u3001\u65e2\u77e5\u307e\u305f\u306f\u7591\u308f\u3057\u3044\u3082\u306e\u3092\u691c\u77e5\u3057\u307e\u3059\u3002
HashLookupModuleFactory.moduleName.text=\u30cf\u30c3\u30b7\u30e5\u30eb\u30c3\u30af\u30a2\u30c3\u30d7
HashLookupModuleFactory.moduleDescription.text=\u6a19\u6e96\u306eNSRL\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u306a\u3069\u3001\u63d0\u4f9b\u3055\u308c\u305f\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u5229\u7528\u3057\u3066\u3001\u65e2\u77e5\u307e\u305f\u306f\u7591\u308f\u3057\u3044\u3082\u306e\u3092\u691c\u77e5\u3057\u307e\u3059\u3002
HashDbIngestModule.noKnownHashDbSetMsg=\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u5b58\u5728\u3057\u307e\u305b\u3093\u3002
HashDbIngestModule.knownFileSearchWillNotExecuteWarn=\u65e2\u77e5\u306e\u30d5\u30a1\u30a4\u30eb\u691c\u7d22\u304c\u5b9f\u884c\u3055\u308c\u307e\u305b\u3093\u3002
HashDbIngestModule.noKnownBadHashDbSetMsg=\u65e2\u77e5\u306e\u60aa\u8cea\u306a\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u30bb\u30c3\u30c8\u306f\u3042\u308a\u307e\u305b\u3093\u3002
@@ -86,7 +86,6 @@ HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=\u65e2\u77e5\u306e\u60aa
HashDbIngestModule.fileReadErrorMsg=\u8aad\u307f\u8fbc\u307f\u30a8\u30e9\u30fc\uff1a {0}
HashDbIngestModule.calcHashValueErr={0}\u306e\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u8a08\u7b97\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002
HashDbIngestModule.hashLookupErrorMsg=\u30cf\u30c3\u30b7\u30e5\u30eb\u30c3\u30af\u30a2\u30c3\u30d7\u30a8\u30e9\u30fc\uff1a {0}
HashDbIngestModule.settingKnownBadStateErr={0}\u306e\u65e2\u77e5\u306e\u60aa\u8cea\u30b9\u30c6\u30fc\u30bf\u30b9\u3092\u8a2d\u5b9a\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002
HashDbIngestModule.lookingUpKnownBadHashValueErr={0}\u306e\u65e2\u77e5\u306e\u60aa\u8cea\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u30eb\u30c3\u30af\u30a2\u30c3\u30d7\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002
HashDbIngestModule.lookingUpKnownHashValueErr={0}\u306e\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u30eb\u30c3\u30af\u30a2\u30c3\u30d7\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002
HashDbIngestModule.postToBB.fileName=\u30d5\u30a1\u30a4\u30eb\u540d

134
Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java
View File

@@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011 - 2013 Basis Technology Corp.
* Copyright 2011-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -53,7 +53,10 @@ import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskException;
@NbBundle.Messages({
/**
* File ingest module to mark files based on hash values.
*/
@Messages({
"HashDbIngestModule.noKnownBadHashDbSetMsg=No notable hash set.",
"HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Notable file search will not be executed.",
"HashDbIngestModule.noKnownHashDbSetMsg=No known hash set.",
@@ -67,18 +70,21 @@ public class HashDbIngestModule implements FileIngestModule {
private final SleuthkitCase skCase;
private final HashDbManager hashDbManager = HashDbManager.getInstance();
private final HashLookupModuleSettings settings;
private List<HashDb> knownBadHashSets = new ArrayList<>();
private List<HashDb> knownHashSets = new ArrayList<>();
private final List<HashDb> knownBadHashSets = new ArrayList<>();
private final List<HashDb> knownHashSets = new ArrayList<>();
private long jobId;
private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
private Blackboard blackboard;
/**
* A container of values for storing ingest metrics for the job.
*/
private static class IngestJobTotals {
private AtomicLong totalKnownBadCount = new AtomicLong(0);
private AtomicLong totalCalctime = new AtomicLong(0);
private AtomicLong totalLookuptime = new AtomicLong(0);
private final AtomicLong totalKnownBadCount = new AtomicLong(0);
private final AtomicLong totalCalctime = new AtomicLong(0);
private final AtomicLong totalLookuptime = new AtomicLong(0);
}
private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
@@ -90,6 +96,15 @@ public class HashDbIngestModule implements FileIngestModule {
return totals;
}
/**
* Create a HashDbIngestModule object that will mark files based on a
* supplied list of hash values. The supplied HashLookupModuleSettings
* object is used to configure the module.
*
* @param settings The module settings.
*
* @throws NoCurrentCaseException If there is no open case.
*/
HashDbIngestModule(HashLookupModuleSettings settings) throws NoCurrentCaseException {
this.settings = settings;
skCase = Case.getCurrentCaseThrows().getSleuthkitCase();
@@ -140,12 +155,18 @@ public class HashDbIngestModule implements FileIngestModule {
enabledHashSets.add(db);
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Error getting index status for " + db.getDisplayName()+ " hash set", ex); //NON-NLS
logger.log(Level.WARNING, "Error getting index status for " + db.getDisplayName() + " hash set", ex); //NON-NLS
}
}
}
}
@Messages({
"# {0} - File name",
"HashDbIngestModule.dialogTitle.errorFindingArtifacts=Error Finding Artifacts: {0}",
"# {0} - File name",
"HashDbIngestModule.errorMessage.lookingForFileArtifacts=Error encountered while looking for existing artifacts for {0}."
})
@Override
public ProcessResult process(AbstractFile file) {
try {
@@ -156,8 +177,8 @@ public class HashDbIngestModule implements FileIngestModule {
}
// Skip unallocated space files.
if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) ||
file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
|| file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
return ProcessResult.OK;
}
@@ -181,6 +202,7 @@ public class HashDbIngestModule implements FileIngestModule {
// calc hash value
String name = file.getName();
long fileId = file.getId();
String md5Hash = file.getMd5Hash();
if (md5Hash == null || md5Hash.isEmpty()) {
try {
@@ -203,15 +225,11 @@ public class HashDbIngestModule implements FileIngestModule {
totals.totalCalctime.addAndGet(delta);
} catch (IOException ex) {
logger.log(Level.WARNING, "Error calculating hash of file " + name, ex); //NON-NLS
logger.log(Level.WARNING, String.format("Error calculating hash of file '%s' (id=%d).", name, fileId), ex); //NON-NLS
services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.fileReadErrorMsg",
name),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.calcHashValueErr",
name)));
NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", name),
NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr", name)));
return ProcessResult.ERROR;
}
}
@@ -245,21 +263,37 @@ public class HashDbIngestModule implements FileIngestModule {
}
}
postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
/*
* We have a match. Now create an artifact if it is
* determined that one hasn't been created yet.
*/
List<BlackboardAttribute> attributesList = new ArrayList<>();
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, HashLookupModuleFactory.getModuleName(), hashSetName));
try {
org.sleuthkit.datamodel.Blackboard tskBlackboard = skCase.getBlackboard();
if (tskBlackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT, attributesList) == false) {
postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
}
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format(
"A problem occurred while checking for existing artifacts for file '%s' (id=%d).", name, fileId), ex); //NON-NLS
services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
Bundle.HashDbIngestModule_dialogTitle_errorFindingArtifacts(name),
Bundle.HashDbIngestModule_errorMessage_lookingForFileArtifacts(name)));
ret = ProcessResult.ERROR;
}
}
long delta = (System.currentTimeMillis() - lookupstart);
totals.totalLookuptime.addAndGet(delta);
} catch (TskException ex) {
logger.log(Level.WARNING, "Couldn't lookup notable hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
logger.log(Level.WARNING, String.format(
"Couldn't lookup notable hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex); //NON-NLS
services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg",
name),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.lookingUpKnownBadHashValueErr",
name)));
NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name),
NbBundle.getMessage(this.getClass(), "HashDbIngestModule.lookingUpKnownBadHashValueErr", name)));
ret = ProcessResult.ERROR;
}
}
@@ -279,15 +313,12 @@ public class HashDbIngestModule implements FileIngestModule {
totals.totalLookuptime.addAndGet(delta);
} catch (TskException ex) {
logger.log(Level.WARNING, "Couldn't lookup known hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
logger.log(Level.WARNING, String.format(
"Couldn't lookup known hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex); //NON-NLS
services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg",
name),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.lookingUpKnownHashValueErr",
name)));
NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name),
NbBundle.getMessage(this.getClass(), "HashDbIngestModule.lookingUpKnownHashValueErr", name)));
ret = ProcessResult.ERROR;
}
}
@@ -296,18 +327,29 @@ public class HashDbIngestModule implements FileIngestModule {
return ret;
}
@Messages({"HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."})
/**
* Post a hash set hit to the blackboard.
*
* @param abstractFile The file to be processed.
* @param md5Hash The MD5 hash value of the file.
* @param hashSetName The name of the hash set with which to associate
* the hit.
* @param comment A comment to be attached to the artifact.
* @param showInboxMessage Show a message in the inbox?
*/
@Messages({
"HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."
})
private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) {
try {
String MODULE_NAME = NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.moduleName");
String moduleName = HashLookupModuleFactory.getModuleName();
BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
Collection<BlackboardAttribute> attributes = new ArrayList<>();
//TODO Revisit usage of deprecated constructor as per TSK-583
//BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName);
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, hashSetName));
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, MODULE_NAME, md5Hash));
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, comment));
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, moduleName, hashSetName));
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, moduleName, md5Hash));
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, comment));
badFile.addAttributes(attributes);
@@ -351,19 +393,24 @@ public class HashDbIngestModule implements FileIngestModule {
detailsSb.append("</table>"); //NON-NLS
services.postMessage(IngestMessage.createDataMessage(HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.postToBB.knownBadMsg",
abstractFile.getName()),
NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.knownBadMsg", abstractFile.getName()),
detailsSb.toString(),
abstractFile.getName() + md5Hash,
badFile));
}
services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_HASHSET_HIT, Collections.singletonList(badFile)));
services.fireModuleDataEvent(new ModuleDataEvent(moduleName, ARTIFACT_TYPE.TSK_HASHSET_HIT, Collections.singletonList(badFile)));
} catch (TskException ex) {
logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS
}
}
/**
* Post a message summarizing the results of the ingest.
*
* @param jobId The ID of the job.
* @param knownBadHashSets The list of hash sets for "known bad" files.
* @param knownHashSets The list of hash sets for "known" files.
*/
private static synchronized void postSummary(long jobId,
List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
IngestJobTotals jobTotals = getTotalsForIngestJobs(jobId);
@@ -399,8 +446,7 @@ public class HashDbIngestModule implements FileIngestModule {
IngestServices.getInstance().postMessage(IngestMessage.createMessage(
IngestMessage.MessageType.INFO,
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(HashDbIngestModule.class,
"HashDbIngestModule.complete.hashLookupResults"),
NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.hashLookupResults"),
detailsSb.toString()));
}
}

13
Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupModuleFactory.java
View File

@@ -42,13 +42,18 @@ public class HashLookupModuleFactory extends IngestModuleFactoryAdapter {
return getModuleName();
}
/**
* Get the name of the module.
*
* @return The module name.
*/
static String getModuleName() {
return NbBundle.getMessage(HashLookupModuleFactory.class, "HashDbIngestModule.moduleName");
return NbBundle.getMessage(HashLookupModuleFactory.class, "HashLookupModuleFactory.moduleName.text");
}
@Override
public String getModuleDescription() {
return NbBundle.getMessage(HashLookupModuleFactory.class, "HashDbIngestModule.moduleDescription");
return NbBundle.getMessage(HashLookupModuleFactory.class, "HashLookupModuleFactory.moduleDescription.text");
}
@Override
@@ -101,8 +106,8 @@ public class HashLookupModuleFactory extends IngestModuleFactoryAdapter {
@Override
public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) {
if (!(settings instanceof HashLookupModuleSettings)) {
throw new IllegalArgumentException(
NbBundle.getMessage(this.getClass(), "HashLookupModuleFactory.createFileIngestModule.exception.msg"));
throw new IllegalArgumentException(NbBundle.getMessage(this.getClass(),
"HashLookupModuleFactory.createFileIngestModule.exception.msg"));
}
try {
return new HashDbIngestModule((HashLookupModuleSettings) settings);