codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-10-03 07:49:52 +00:00
Code Issues Releases Wiki Activity

Preventing duplicate keyword hits.

Browse Source
This commit is contained in:
U-BASIS\dgrove
2018-07-30 10:52:47 -04:00
parent 55f60073dd
commit 490d8df3e7
3 changed files with 87 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
+39 -24
View File
@@ -32,6 +32,8 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CursorMarkParams;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.Version;
@@ -40,6 +42,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskException;
@@ -203,15 +206,44 @@ class LuceneQuery implements KeywordSearchQuery {
* @param listName The name of the keyword list that contained the
* keyword for which the hit was found.
*
*
* @return The newly created artifact or null if there was a problem
* creating it.
* @return The newly created artifact, or null if one wasn't created due to
* either the artifact already existing or an error while trying to
* create it.
*/
@Override
public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) {
final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
Collection<BlackboardAttribute> attributes = new ArrayList<>();
List<BlackboardAttribute> attributesList = new ArrayList<>();
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
if (originalKeyword != null) {
BlackboardAttribute.ATTRIBUTE_TYPE selType = originalKeyword.getArtifactAttributeType();
if (selType != null) {
attributesList.add(new BlackboardAttribute(selType, MODULE_NAME, foundKeyword.getSearchTerm()));
}
if (originalKeyword.searchTermIsWholeWord()) {
attributesList.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal()));
} else {
attributesList.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal()));
}
}
if (StringUtils.isNotBlank(listName)) {
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
}
try {
SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase();
org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard();
if (tskBlackboard.artifactExists(content, BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT, attributesList)) {
return null;
}
} catch (NoCurrentCaseException | TskCoreException ex) {
logger.log(Level.SEVERE, String.format(
"A problem occurred while checking for existing artifacts for file '%s' (id=%d).",
content.getName(), content.getId()), ex); //NON-NLS
}
BlackboardArtifact bba;
try {
bba = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
@@ -221,32 +253,15 @@ class LuceneQuery implements KeywordSearchQuery {
}
if (snippet != null) {
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
}
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
if (StringUtils.isNotBlank(listName)) {
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
}
if (originalKeyword != null) {
BlackboardAttribute.ATTRIBUTE_TYPE selType = originalKeyword.getArtifactAttributeType();
if (selType != null) {
attributes.add(new BlackboardAttribute(selType, MODULE_NAME, foundKeyword.getSearchTerm()));
}
if (originalKeyword.searchTermIsWholeWord()) {
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal()));
} else {
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal()));
}
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
}
hit.getArtifactID().ifPresent(artifactID
-> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
-> attributesList.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
);
try {
bba.addAttributes(attributes); //write out to bb
bba.addAttributes(attributesList); //write out to bb
return bba;
} catch (TskCoreException e) {
logger.log(Level.WARNING, "Error adding bb attributes to artifact", e); //NON-NLS
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java
+9 -2
View File
@@ -41,6 +41,7 @@ import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
@@ -65,7 +66,7 @@ class QueryResults {
* and publishing an event to notify subscribers of the blackboard posts.
*
* The KeywordSearchQuery is used to do the blackboard posts.
*
*
* @param query The query.
*/
QueryResults(KeywordSearchQuery query) {
@@ -220,8 +221,14 @@ class QueryResults {
}
/*
* Post an artifact for the hit to the blackboard.
* Create an artifact if it is determined that one hasn't been
* created yet.
*/
if (content == null) {
logger.log(Level.WARNING, "Cannot add artifact for keyword hit to blackboard without a Content object."); //NON-NLS
continue; // Cycle to the next KeywordHit.
}
BlackboardArtifact artifact = query.postKeywordHitToBlackboard(content, keyword, hit, snippet, query.getKeywordList().getName());
/*
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java
+39 -27
View File
@@ -55,6 +55,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
@@ -74,7 +75,7 @@ import org.sleuthkit.datamodel.TskData;
*/
final class RegexQuery implements KeywordSearchQuery {
public static final Logger LOGGER = Logger.getLogger(RegexQuery.class.getName());
public static final Logger logger = Logger.getLogger(RegexQuery.class.getName());
/**
* Lucene regular expressions do not support the following Java predefined
@@ -213,7 +214,7 @@ final class RegexQuery implements KeywordSearchQuery {
hitsForKeyword.add(hit);
}
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error creating keyword hits", ex); //NON-NLS
logger.log(Level.SEVERE, "Error creating keyword hits", ex); //NON-NLS
}
}
@@ -223,7 +224,7 @@ final class RegexQuery implements KeywordSearchQuery {
}
cursorMark = nextCursorMark;
} catch (KeywordSearchModuleException ex) {
LOGGER.log(Level.SEVERE, "Error executing Regex Solr Query: " + keywordString, ex); //NON-NLS
logger.log(Level.SEVERE, "Error executing Regex Solr Query: " + keywordString, ex); //NON-NLS
MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.query.exception.msg", keywordString), ex.getCause().getMessage());
}
}
@@ -437,18 +438,38 @@ final class RegexQuery implements KeywordSearchQuery {
* @param listName The name of the keyword list that contained the
* keyword for which the hit was found.
*
*
* @return The newly created artifact or null if there was a problem
* creating it.
* @return The newly created artifact, or null if one wasn't created due to
* either the artifact already existing or an error while trying to
* create it.
*/
@Override
public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) {
final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
if (content == null) {
LOGGER.log(Level.WARNING, "Error adding artifact for keyword hit to blackboard"); //NON-NLS
logger.log(Level.WARNING, "Error adding artifact for keyword hit to blackboard"); //NON-NLS
return null;
}
List<BlackboardAttribute> attributesList = new ArrayList<>();
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal()));
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString()));
if (StringUtils.isNotBlank(listName)) {
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
}
try {
SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase();
org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard();
if (tskBlackboard.artifactExists(content, BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT, attributesList)) {
return null;
}
} catch (NoCurrentCaseException | TskCoreException ex) {
logger.log(Level.SEVERE, String.format(
"A problem occurred while checking for existing artifacts for file '%s' (id=%d).",
content.getName(), content.getId()), ex); //NON-NLS
}
/*
* Credit Card number hits are handled differently
@@ -463,36 +484,27 @@ final class RegexQuery implements KeywordSearchQuery {
* regex attributes
*/
BlackboardArtifact newArtifact;
Collection<BlackboardAttribute> attributes = new ArrayList<>();
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString()));
try {
newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS
logger.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS
return null;
}
if (StringUtils.isNotBlank(listName)) {
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
}
if (snippet != null) {
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
}
hit.getArtifactID().ifPresent(artifactID
-> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
-> attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
);
attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal()));
try {
newArtifact.addAttributes(attributes);
newArtifact.addAttributes(attributesList);
return newArtifact;
} catch (TskCoreException e) {
LOGGER.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS
logger.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS
return null;
}
}
@@ -502,7 +514,7 @@ final class RegexQuery implements KeywordSearchQuery {
final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
LOGGER.log(Level.SEVERE, "Keyword hit is not a credit card number"); //NON-NLS
logger.log(Level.SEVERE, "Keyword hit is not a credit card number"); //NON-NLS
return;
}
/*
@@ -525,13 +537,13 @@ final class RegexQuery implements KeywordSearchQuery {
if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) {
if (hit.isArtifactHit()) {
LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS
logger.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS
} else {
try {
LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getContentID())); //NON-NLS
logger.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getContentID())); //NON-NLS
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s' ", foundKeyword.getSearchTerm(), hit.getSnippet())); //NON-NLS
LOGGER.log(Level.SEVERE, "There was a error getting contentID for keyword hit.", ex); //NON-NLS
logger.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s' ", foundKeyword.getSearchTerm(), hit.getSnippet())); //NON-NLS
logger.log(Level.SEVERE, "There was a error getting contentID for keyword hit.", ex); //NON-NLS
}
}
return;
@@ -599,7 +611,7 @@ final class RegexQuery implements KeywordSearchQuery {
ccAccountInstance.addAttributes(attributes);
} catch (TskCoreException | NoCurrentCaseException ex) {
LOGGER.log(Level.SEVERE, "Error creating CCN account instance", ex); //NON-NLS
logger.log(Level.SEVERE, "Error creating CCN account instance", ex); //NON-NLS
}