From 490d8df3e7fdfd5443a5bf19cceaeb9f3217ab90 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 30 Jul 2018 10:52:47 -0400 Subject: [PATCH] Preventing duplicate keyword hits. --- .../autopsy/keywordsearch/LuceneQuery.java | 63 +++++++++++------- .../autopsy/keywordsearch/QueryResults.java | 11 +++- .../autopsy/keywordsearch/RegexQuery.java | 66 +++++++++++-------- 3 files changed, 87 insertions(+), 53 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index e13531da26..fadc2f5107 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -32,6 +32,8 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.CursorMarkParams; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Version; @@ -40,6 +42,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskException; @@ -203,15 +206,44 @@ class LuceneQuery implements KeywordSearchQuery { * @param listName The name of the keyword list that contained the * keyword for which the hit was found. * - * - * @return The newly created artifact or null if there was a problem - * creating it. + * @return The newly created artifact, or null if one wasn't created due to + * either the artifact already existing or an error while trying to + * create it. */ @Override public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); - Collection attributes = new ArrayList<>(); + List attributesList = new ArrayList<>(); + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); + if (originalKeyword != null) { + BlackboardAttribute.ATTRIBUTE_TYPE selType = originalKeyword.getArtifactAttributeType(); + if (selType != null) { + attributesList.add(new BlackboardAttribute(selType, MODULE_NAME, foundKeyword.getSearchTerm())); + } + + if (originalKeyword.searchTermIsWholeWord()) { + attributesList.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal())); + } else { + attributesList.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal())); + } + } + if (StringUtils.isNotBlank(listName)) { + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); + } + + try { + SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); + org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard(); + if (tskBlackboard.artifactExists(content, BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT, attributesList)) { + return null; + } + } catch (NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, String.format( + "A problem occurred while checking for existing artifacts for file '%s' (id=%d).", + content.getName(), content.getId()), ex); //NON-NLS + } + BlackboardArtifact bba; try { bba = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); @@ -221,32 +253,15 @@ class LuceneQuery implements KeywordSearchQuery { } if (snippet != null) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); - } - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); - if (StringUtils.isNotBlank(listName)) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); - } - - if (originalKeyword != null) { - BlackboardAttribute.ATTRIBUTE_TYPE selType = originalKeyword.getArtifactAttributeType(); - if (selType != null) { - attributes.add(new BlackboardAttribute(selType, MODULE_NAME, foundKeyword.getSearchTerm())); - } - - if (originalKeyword.searchTermIsWholeWord()) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal())); - } else { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal())); - } + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); } hit.getArtifactID().ifPresent(artifactID - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) + -> attributesList.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) ); try { - bba.addAttributes(attributes); //write out to bb + bba.addAttributes(attributesList); //write out to bb return bba; } catch (TskCoreException e) { logger.log(Level.WARNING, "Error adding bb attributes to artifact", e); //NON-NLS diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index e27a01b063..3b7dd81fa0 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -41,6 +41,7 @@ import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -65,7 +66,7 @@ class QueryResults { * and publishing an event to notify subscribers of the blackboard posts. * * The KeywordSearchQuery is used to do the blackboard posts. - * + * * @param query The query. */ QueryResults(KeywordSearchQuery query) { @@ -220,8 +221,14 @@ class QueryResults { } /* - * Post an artifact for the hit to the blackboard. + * Create an artifact if it is determined that one hasn't been + * created yet. */ + if (content == null) { + logger.log(Level.WARNING, "Cannot add artifact for keyword hit to blackboard without a Content object."); //NON-NLS + continue; // Cycle to the next KeywordHit. + } + BlackboardArtifact artifact = query.postKeywordHitToBlackboard(content, keyword, hit, snippet, query.getKeywordList().getName()); /* diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index cb640e20bd..1706cdef60 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -55,6 +55,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -74,7 +75,7 @@ import org.sleuthkit.datamodel.TskData; */ final class RegexQuery implements KeywordSearchQuery { - public static final Logger LOGGER = Logger.getLogger(RegexQuery.class.getName()); + public static final Logger logger = Logger.getLogger(RegexQuery.class.getName()); /** * Lucene regular expressions do not support the following Java predefined @@ -213,7 +214,7 @@ final class RegexQuery implements KeywordSearchQuery { hitsForKeyword.add(hit); } } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error creating keyword hits", ex); //NON-NLS + logger.log(Level.SEVERE, "Error creating keyword hits", ex); //NON-NLS } } @@ -223,7 +224,7 @@ final class RegexQuery implements KeywordSearchQuery { } cursorMark = nextCursorMark; } catch (KeywordSearchModuleException ex) { - LOGGER.log(Level.SEVERE, "Error executing Regex Solr Query: " + keywordString, ex); //NON-NLS + logger.log(Level.SEVERE, "Error executing Regex Solr Query: " + keywordString, ex); //NON-NLS MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.query.exception.msg", keywordString), ex.getCause().getMessage()); } } @@ -437,18 +438,38 @@ final class RegexQuery implements KeywordSearchQuery { * @param listName The name of the keyword list that contained the * keyword for which the hit was found. * - * - * @return The newly created artifact or null if there was a problem - * creating it. + * @return The newly created artifact, or null if one wasn't created due to + * either the artifact already existing or an error while trying to + * create it. */ @Override public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); if (content == null) { - LOGGER.log(Level.WARNING, "Error adding artifact for keyword hit to blackboard"); //NON-NLS + logger.log(Level.WARNING, "Error adding artifact for keyword hit to blackboard"); //NON-NLS return null; } + + List attributesList = new ArrayList<>(); + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString())); + if (StringUtils.isNotBlank(listName)) { + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); + } + + try { + SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); + org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard(); + if (tskBlackboard.artifactExists(content, BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT, attributesList)) { + return null; + } + } catch (NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, String.format( + "A problem occurred while checking for existing artifacts for file '%s' (id=%d).", + content.getName(), content.getId()), ex); //NON-NLS + } /* * Credit Card number hits are handled differently @@ -463,36 +484,27 @@ final class RegexQuery implements KeywordSearchQuery { * regex attributes */ BlackboardArtifact newArtifact; - Collection attributes = new ArrayList<>(); - - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString())); try { newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS + logger.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS return null; } - if (StringUtils.isNotBlank(listName)) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); - } if (snippet != null) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); + attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); } hit.getArtifactID().ifPresent(artifactID - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) + -> attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) ); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); - try { - newArtifact.addAttributes(attributes); + newArtifact.addAttributes(attributesList); return newArtifact; } catch (TskCoreException e) { - LOGGER.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS + logger.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS return null; } } @@ -502,7 +514,7 @@ final class RegexQuery implements KeywordSearchQuery { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { - LOGGER.log(Level.SEVERE, "Keyword hit is not a credit card number"); //NON-NLS + logger.log(Level.SEVERE, "Keyword hit is not a credit card number"); //NON-NLS return; } /* @@ -525,13 +537,13 @@ final class RegexQuery implements KeywordSearchQuery { if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) { if (hit.isArtifactHit()) { - LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS + logger.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS } else { try { - LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getContentID())); //NON-NLS + logger.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getContentID())); //NON-NLS } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s' ", foundKeyword.getSearchTerm(), hit.getSnippet())); //NON-NLS - LOGGER.log(Level.SEVERE, "There was a error getting contentID for keyword hit.", ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s' ", foundKeyword.getSearchTerm(), hit.getSnippet())); //NON-NLS + logger.log(Level.SEVERE, "There was a error getting contentID for keyword hit.", ex); //NON-NLS } } return; @@ -599,7 +611,7 @@ final class RegexQuery implements KeywordSearchQuery { ccAccountInstance.addAttributes(attributes); } catch (TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.SEVERE, "Error creating CCN account instance", ex); //NON-NLS + logger.log(Level.SEVERE, "Error creating CCN account instance", ex); //NON-NLS }