From a50cf3da86b93788e884c10aa6feaa25f6268bf7 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 11:36:04 -0400 Subject: [PATCH 1/4] User searches appear in report (no heading yet) --- .../autopsy/report/TableReportGenerator.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 9d5d41c836..8c80955b23 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -555,6 +555,7 @@ class TableReportGenerator { "AND att.artifact_id = art.artifact_id " + //NON-NLS "GROUP BY list " + orderByClause; //NON-NLS + System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); @@ -589,8 +590,9 @@ class TableReportGenerator { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } // Query for keywords, grouped by list - String keywordsQuery - = "SELECT art.artifact_id, art.obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + // Query for keywords that are part of a list + String keywordListsQuery + = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + //NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, blackboard_attributes AS att3, tsk_files AS f " + //NON-NLS @@ -608,9 +610,26 @@ class TableReportGenerator { + //NON-NLS "AND (att3.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ") " + //NON-NLS - "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " - + //NON-NLS - orderByClause; //NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; + + // Query for keywords that are not part of a list + String keywordAdHocQuery = + "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS + "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS + "WHERE " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) " + // NON-NLS + "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) AND (att1.attribute_type_id = 37))) " + //NON-NLS + "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS + "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS + "AND (f.obj_id = art.obj_id) " + //NON-NLS + "AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID() + ") " + // NON-NLS + "AND (att2.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID() + ") " + // NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + + String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; + + System.out.println("\n\n### kw hits query:" + keywordsQuery); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); From 498c19c9dc6e4d2a9503d1ceaf99b083f137c133 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 13:20:45 -0400 Subject: [PATCH 2/4] Add fix for notable tags. --- .../autopsy/report/TableReportGenerator.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 8c80955b23..e8d8b067b8 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import static org.sleuthkit.autopsy.casemodule.services.TagsManager.getNotableTagLabel; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.Type; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -538,6 +540,32 @@ class TableReportGenerator { logger.log(Level.SEVERE, "Exception while getting open case: ", ex); //NON-NLS return; } + + // First check for ad-hoc keywords + String tagIDList = ""; + if(tagNamesFilter.isEmpty()) { + System.out.println("\n## no tags selected"); + } else { + System.out.println("\n## tags: " + makeCommaSeparatedList(tagNamesFilter)); + try { + Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); + for(String tagDisplayName : tagNamesFilter) { + if(tagNamesMap.containsKey(tagDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(tagDisplayName).getId(); + } else { + System.out.println("### No key for " + tagDisplayName); + } + } + } catch (NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, "Exception while getting tag info - proceeding without tag filter: ", ex); //NON-NLS + tagIDList = ""; + } + } + System.out.println("### tag ID list: " + tagIDList); + if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -546,15 +574,18 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art, blackboard_artifact_tags as tag " + //NON-NLS "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS - "AND att.artifact_id = art.artifact_id " - + //NON-NLS - "GROUP BY list " + orderByClause; //NON-NLS + "AND att.artifact_id = art.artifact_id "; + if (! tagIDList.isEmpty()) { + keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS + "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { @@ -1642,14 +1673,15 @@ class TableReportGenerator { private HashSet getUniqueTagNames(long artifactId) throws TskCoreException { HashSet uniqueTagNames = new HashSet<>(); - String query = "SELECT display_name, artifact_id FROM tag_names AS tn, blackboard_artifact_tags AS bat " + String query = "SELECT display_name, artifact_id, knownStatus FROM tag_names AS tn, blackboard_artifact_tags AS bat " + //NON-NLS "WHERE tn.tag_name_id = bat.tag_name_id AND bat.artifact_id = " + artifactId; //NON-NLS try (SleuthkitCase.CaseDbQuery dbQuery = Case.getCurrentCaseThrows().getSleuthkitCase().executeQuery(query)) { ResultSet tagNameRows = dbQuery.getResultSet(); while (tagNameRows.next()) { - uniqueTagNames.add(tagNameRows.getString("display_name")); //NON-NLS + String notableString = tagNameRows.getInt("knownStatus") == TskData.FileKnown.BAD.ordinal() ? getNotableTagLabel() : ""; + uniqueTagNames.add(tagNameRows.getString("display_name") + notableString); //NON-NLS } } catch (TskCoreException | SQLException | NoCurrentCaseException ex) { throw new TskCoreException("Error getting tag names for artifact: ", ex); From 9ef33ab771bf8343d062b934fc6ee6938bf67d21 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 14:16:35 -0400 Subject: [PATCH 3/4] Added user searches to table of contents when applicable Added fix for Notable tags Cleanup --- .../autopsy/report/TableReportGenerator.java | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index e8d8b067b8..59951df1f9 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -541,12 +541,9 @@ class TableReportGenerator { return; } - // First check for ad-hoc keywords + // Get a list of all selected tag IDs String tagIDList = ""; - if(tagNamesFilter.isEmpty()) { - System.out.println("\n## no tags selected"); - } else { - System.out.println("\n## tags: " + makeCommaSeparatedList(tagNamesFilter)); + if( ! tagNamesFilter.isEmpty()) { try { Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); for(String tagDisplayName : tagNamesFilter) { @@ -556,7 +553,16 @@ class TableReportGenerator { } tagIDList += tagNamesMap.get(tagDisplayName).getId(); } else { - System.out.println("### No key for " + tagDisplayName); + // If the tag name ends with "(Notable)", try stripping that off + if(tagDisplayName.endsWith(getNotableTagLabel())) { + String editedDisplayName = tagDisplayName.substring(0, tagDisplayName.length() - getNotableTagLabel().length()); + if(tagNamesMap.containsKey(editedDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(editedDisplayName).getId(); + } + } } } } catch (NoCurrentCaseException | TskCoreException ex) { @@ -564,8 +570,35 @@ class TableReportGenerator { tagIDList = ""; } } - System.out.println("### tag ID list: " + tagIDList); + // Check if there are any ad-hoc results + String adHocCountQuery = "SELECT COUNT(*) FROM " + //NON-NLS + "(SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 ";//NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + adHocCountQuery += "WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += " AND (art.artifact_id = tag.artifact_id) AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + adHocCountQuery += "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ")) "; //NON-NLS + + int adHocCount = 0; + try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(adHocCountQuery)) { + ResultSet adHocCountResultSet = dbQuery.getResultSet(); + if (adHocCountResultSet.next()) { + adHocCount = adHocCountResultSet.getInt(1); //NON-NLS + } else { + throw new TskCoreException("Error counting ad hoc keywords"); + } + } catch (TskCoreException | SQLException ex) { + errorList.add(NbBundle.getMessage(this.getClass(), "ReportGenerator.errList.failedQueryKWLists")); + logger.log(Level.SEVERE, "Failed to count ad hoc searches with query " + adHocCountQuery, ex); //NON-NLS + return; + } + + // Create the query to get the keyword list names if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -585,9 +618,12 @@ class TableReportGenerator { keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS } + if (adHocCount > 0) { + keywordListQuery += " UNION SELECT \"\" AS list "; + } keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS - System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); + // Make the table of contents links for each list type try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); List lists = new ArrayList<>(); @@ -611,6 +647,7 @@ class TableReportGenerator { return; } + // Query for keywords, grouped by list if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att3.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + "convert_to(att1.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS @@ -620,7 +657,7 @@ class TableReportGenerator { } else { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } - // Query for keywords, grouped by list + // Query for keywords that are part of a list String keywordListsQuery = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " @@ -648,9 +685,9 @@ class TableReportGenerator { "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS "WHERE " + // NON-NLS - " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " + // NON-NLS "EXCEPT " + // NON-NLS - "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) AND (att1.attribute_type_id = 37))) " + //NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + "))) " + //NON-NLS "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS "AND (f.obj_id = art.obj_id) " + //NON-NLS @@ -659,8 +696,6 @@ class TableReportGenerator { "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; - - System.out.println("\n\n### kw hits query:" + keywordsQuery); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); From 414a65402aed9dc6e269ef469ca68fa7a9f8106b Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 14:30:21 -0400 Subject: [PATCH 4/4] Cleanup --- .../sleuthkit/autopsy/report/TableReportGenerator.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 59951df1f9..f87851fe87 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -607,9 +607,11 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art, blackboard_artifact_tags as tag " - + //NON-NLS - "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art "; // NON-NLS + if(! tagIDList.isEmpty()) { + keywordListQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + keywordListQuery += "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS