From e8ed08ec93a714346c3d00119887da3c7705a725 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 30 Nov 2021 14:58:04 -0500 Subject: [PATCH 01/24] add type id; need to integrate in DAOs and tree --- .../mainui/datamodel/AccountSearchParams.java | 10 ++++- .../datamodel/AnalysisResultSearchParam.java | 13 +++++- .../AnalysisResultSetSearchParam.java | 14 +++++-- .../BlackboardArtifactSearchParam.java | 10 +++++ .../datamodel/CommAccountsSearchParams.java | 11 ++++- .../datamodel/DataArtifactSearchParam.java | 13 +++++- .../FileSystemContentSearchParam.java | 10 +++++ .../datamodel/FileSystemHostSearchParam.java | 11 ++++- .../FileSystemPersonSearchParam.java | 15 ++++++- .../FileTypeExtensionsSearchParams.java | 9 ++++ .../datamodel/FileTypeMimeSearchParams.java | 12 +++++- .../datamodel/FileTypeSizeSearchParams.java | 8 ++++ .../mainui/datamodel/HashHitSearchParam.java | 11 ++++- .../datamodel/KeywordHitSearchParam.java | 18 +++++--- .../mainui/datamodel/KeywordMatchParams.java | 9 ++++ .../datamodel/KeywordSearchTermParams.java | 9 ++++ .../datamodel/OsAccountsSearchParams.java | 11 ++++- .../mainui/datamodel/TagsSearchParams.java | 13 +++++- .../nodes/AnalysisResultTypeFactory.java | 41 +++++++++++++++++-- .../mainui/nodes/FileSystemFactory.java | 4 -- .../mainui/nodes/ViewsTypeFactory.java | 3 -- 21 files changed, 221 insertions(+), 34 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AccountSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AccountSearchParams.java index b9959f80c5..119aa4c9b7 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AccountSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AccountSearchParams.java @@ -26,7 +26,15 @@ import org.sleuthkit.datamodel.BlackboardArtifact; * Search parameters for accounts. */ public class AccountSearchParams extends DataArtifactSearchParam { - + private static final String TYPE_ID = "DATA_ARTIFACT_ACCOUNT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final Account.Type accountType; /** diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java index 4f2805a650..aca06e0081 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java @@ -24,8 +24,17 @@ import org.sleuthkit.datamodel.BlackboardArtifact; * Key for analysis result in order to retrieve data from DAO. */ public class AnalysisResultSearchParam extends BlackboardArtifactSearchParam { - + + private static final String TYPE_ID = "ANALYSIS_RESULT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + public AnalysisResultSearchParam(BlackboardArtifact.Type artifactType, Long dataSourceId) { super(artifactType, dataSourceId); - } + } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSetSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSetSearchParam.java index 767491c3fe..ca4480d708 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSetSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSetSearchParam.java @@ -25,14 +25,23 @@ import org.sleuthkit.datamodel.BlackboardArtifact; * Base class for search params for analysis results that filter by set name. */ public class AnalysisResultSetSearchParam extends AnalysisResultSearchParam { - + + private static final String TYPE_ID = "ANALYSIS_RESULT_SET"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final String setName; public AnalysisResultSetSearchParam(BlackboardArtifact.Type artifactType, Long dataSourceId, String setName) { super(artifactType, dataSourceId); this.setName = setName; } - + public String getSetName() { return setName; } @@ -63,5 +72,4 @@ public class AnalysisResultSetSearchParam extends AnalysisResultSearchParam { return super.equals(obj); } - } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java index 2bfd9a3929..8fd1f38c1d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java @@ -25,6 +25,16 @@ import org.sleuthkit.datamodel.BlackboardArtifact; * Key for data artifact in order to retrieve data from DAO. */ public class BlackboardArtifactSearchParam { + + private static final String TYPE_ID = "BLACKBOARD_ARTIFACT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final BlackboardArtifact.Type artifactType; private final Long dataSourceId; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsSearchParams.java index 91c5e272ae..695deaa0fe 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsSearchParams.java @@ -26,9 +26,18 @@ import org.sleuthkit.datamodel.Account; */ public class CommAccountsSearchParams { + private static final String TYPE_ID = "DATA_ARTIFACT_ACCOUNT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final Account.Type type; private final Long dataSourceId; - + public CommAccountsSearchParams(Account.Type type, Long dataSourceId) { this.type = type; this.dataSourceId = dataSourceId; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java index 0956cf4b5c..cf06b9a356 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java @@ -25,7 +25,16 @@ import org.sleuthkit.datamodel.BlackboardArtifact; */ public class DataArtifactSearchParam extends BlackboardArtifactSearchParam { + private static final String TYPE_ID = "DATA_ARTIFACT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + public DataArtifactSearchParam(BlackboardArtifact.Type artifactType, Long dataSourceId) { - super (artifactType, dataSourceId); - } + super(artifactType, dataSourceId); + } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemContentSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemContentSearchParam.java index ff74c0ab85..32c5798f7a 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemContentSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemContentSearchParam.java @@ -24,6 +24,16 @@ import java.util.Objects; * Key for content object in order to retrieve data from DAO. */ public class FileSystemContentSearchParam { + + private static final String TYPE_ID = "FILE_SYSTEM_CONTENT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final Long contentObjectId; public FileSystemContentSearchParam(Long contentObjectId) { diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemHostSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemHostSearchParam.java index 2f78e597a8..a258b2320d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemHostSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemHostSearchParam.java @@ -24,6 +24,16 @@ import java.util.Objects; * Key for content object in order to retrieve data from DAO. */ public class FileSystemHostSearchParam { + + private static final String TYPE_ID = "FILE_SYSTEM_HOST"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final Long hostObjectId; public FileSystemHostSearchParam(Long hostObjectId) { @@ -59,4 +69,3 @@ public class FileSystemHostSearchParam { return true; } } - diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemPersonSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemPersonSearchParam.java index 99f75b1764..ae198a89dd 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemPersonSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemPersonSearchParam.java @@ -24,12 +24,23 @@ import java.util.Objects; * Key for person object in order to retrieve data from DAO. */ public class FileSystemPersonSearchParam { + + private static final String TYPE_ID = "FILE_SYSTEM_PERSON"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final Long personObjectId; /** * Create search param. - * - * @param personObjectId May be null to fetch hosts not associated with a Person + * + * @param personObjectId May be null to fetch hosts not associated with a + * Person */ public FileSystemPersonSearchParam(Long personObjectId) { this.personObjectId = personObjectId; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeExtensionsSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeExtensionsSearchParams.java index fe875e1463..be0bf45de2 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeExtensionsSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeExtensionsSearchParams.java @@ -25,6 +25,15 @@ import java.util.Objects; */ public class FileTypeExtensionsSearchParams { + private static final String TYPE_ID = "FILE_VIEWS_EXTENSION"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final FileExtSearchFilter filter; private final Long dataSourceId; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeMimeSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeMimeSearchParams.java index 8e95b18beb..0b89294e95 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeMimeSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeMimeSearchParams.java @@ -25,9 +25,18 @@ import java.util.Objects; */ public class FileTypeMimeSearchParams { + private static final String TYPE_ID = "FILE_VIEWS_MIME"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final String mimeType; private final Long dataSourceId; - + public FileTypeMimeSearchParams(String mimeType, Long dataSourceId) { this.mimeType = mimeType; this.dataSourceId = dataSourceId; @@ -70,5 +79,4 @@ public class FileTypeMimeSearchParams { return true; } - } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java index 1a1eb91917..898f1fca83 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java @@ -25,6 +25,14 @@ import java.util.Objects; */ public class FileTypeSizeSearchParams { + private static final String TYPE_ID = "FILE_VIEWS_SIZE"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } private final FileSizeFilter sizeFilter; private final Long dataSourceId; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/HashHitSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/HashHitSearchParam.java index dc8495bffd..96299f9804 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/HashHitSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/HashHitSearchParam.java @@ -24,7 +24,16 @@ import org.sleuthkit.datamodel.BlackboardArtifact; * Key for keyword hits in order to retrieve data from DAO. */ public class HashHitSearchParam extends AnalysisResultSetSearchParam { - + + private static final String TYPE_ID = "HASH_HIT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + public HashHitSearchParam(Long dataSourceId, String setName) { super(BlackboardArtifact.Type.TSK_HASHSET_HIT, dataSourceId, setName); } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordHitSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordHitSearchParam.java index fc59d36bd2..fe9a53f012 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordHitSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordHitSearchParam.java @@ -26,19 +26,28 @@ import org.sleuthkit.datamodel.BlackboardArtifact; */ public class KeywordHitSearchParam extends AnalysisResultSetSearchParam { + private static final String TYPE_ID = "KEYWORD_HIT"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final String keyword; private final String regex; - + public KeywordHitSearchParam(Long dataSourceId, String setName, String keyword, String regex) { super(BlackboardArtifact.Type.TSK_KEYWORD_HIT, dataSourceId, setName); this.keyword = keyword; this.regex = regex; } - + public String getRegex() { return regex; } - + public String getKeyword() { return keyword; } @@ -72,6 +81,5 @@ public class KeywordHitSearchParam extends AnalysisResultSetSearchParam { } return super.equals(obj); } - - + } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordMatchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordMatchParams.java index 1351939423..597c55e7c4 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordMatchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordMatchParams.java @@ -23,6 +23,15 @@ package org.sleuthkit.autopsy.mainui.datamodel; */ public class KeywordMatchParams { + private static final String TYPE_ID = "KEYWORD_MATCH"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final String setName; private final String searchTerm; private final String keywordMatch; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordSearchTermParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordSearchTermParams.java index c22a6c97b9..d2acd4a74d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordSearchTermParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/KeywordSearchTermParams.java @@ -23,6 +23,15 @@ package org.sleuthkit.autopsy.mainui.datamodel; */ public class KeywordSearchTermParams { + private static final String TYPE_ID = "KEYWORD_SEARCH_TERM"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final String setName; private final String searchTerm; private final boolean hasChildren; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsSearchParams.java index e21db61826..61801fbb3d 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsSearchParams.java @@ -25,8 +25,17 @@ import java.util.Objects; */ public class OsAccountsSearchParams { + private static final String TYPE_ID = "OS_ACCOUNTS"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + private final Long dataSourceId; - + public OsAccountsSearchParams(Long dataSourceId) { this.dataSourceId = dataSourceId; } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java index 2f239047cd..5f5d862cf3 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java @@ -25,7 +25,16 @@ import org.sleuthkit.datamodel.TagName; * Key for accessing data about tags from the DAO. */ public class TagsSearchParams { - + + private static final String TYPE_ID = "TAG"; + + /** + * @return The type id for this search parameter. + */ + public static String getTypeId() { + return TYPE_ID; + } + public enum TagType { FILE, RESULT; @@ -34,7 +43,7 @@ public class TagsSearchParams { private final TagType type; private final TagName tagName; private final Long dataSourceId; - + public TagsSearchParams(TagName tagName, TagType type, Long dataSourceId) { this.tagName = tagName; this.type = type; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java index 6c17eb5daf..6de6ec6752 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java @@ -22,6 +22,7 @@ import org.sleuthkit.autopsy.mainui.datamodel.KeywordSearchTermParams; import org.sleuthkit.autopsy.mainui.datamodel.KeywordMatchParams; import com.google.common.collect.ImmutableSet; import java.util.Comparator; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutionException; import org.openide.nodes.ChildFactory; @@ -29,8 +30,11 @@ import org.openide.nodes.Children; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; import org.sleuthkit.autopsy.datamodel.utils.IconsUtil; +import org.sleuthkit.autopsy.mainui.datamodel.AnalysisResultDAO; import org.sleuthkit.autopsy.mainui.datamodel.AnalysisResultSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.AnalysisResultSetSearchParam; +import org.sleuthkit.autopsy.mainui.datamodel.DataArtifactDAO; +import org.sleuthkit.autopsy.mainui.datamodel.DataArtifactSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.KeywordHitSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; @@ -92,8 +96,23 @@ public class AnalysisResultTypeFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent daoEvt) { - // GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, AnalysisResultSearchParam.class); + + if (originalTreeItem != null + && !AnalysisResultDAO.getIgnoredTreeTypes().contains(originalTreeItem.getSearchParams().getArtifactType()) + && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + AnalysisResultSearchParam searchParam = originalTreeItem.getSearchParams(); + return new TreeResultsDTO.TreeItemDTO<>( + BlackboardArtifact.Category.ANALYSIS_RESULT.name(), + new AnalysisResultSearchParam(searchParam.getArtifactType(), searchParam.getDataSourceId()), + searchParam.getArtifactType().getTypeID(), + searchParam.getArtifactType().getDisplayName(), + originalTreeItem.getDisplayCount()); + } return null; } @@ -179,8 +198,22 @@ public class AnalysisResultTypeFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent daoEvt) { - // GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, AnalysisResultSetSearchParam.class); + + if (originalTreeItem != null + && !AnalysisResultDAO.getIgnoredTreeTypes().contains(originalTreeItem.getSearchParams().getArtifactType()) + && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + AnalysisResultSetSearchParam searchParam = originalTreeItem.getSearchParams(); + return new TreeResultsDTO.TreeItemDTO<>( + TBD, + new AnalysisResultSetSearchParam(searchParam.getArtifactType(), searchParam.getDataSourceId(), searchParam.getSetName()), + searchParam.getArtifactType().getTypeID(), + searchParam.getArtifactType().getDisplayName(), + originalTreeItem.getDisplayCount()); + } return null; } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java index 1d37b9a889..e7871526e2 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.mainui.nodes; -import java.beans.PropertyChangeEvent; import java.util.Optional; import org.openide.nodes.Children; import org.openide.nodes.Node; @@ -27,7 +26,6 @@ import java.util.logging.Level; import javax.swing.Action; import org.openide.util.Lookup; import org.openide.util.NbBundle; -import org.python.google.common.primitives.Longs; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; @@ -35,8 +33,6 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.FileTypeExtensions; import org.sleuthkit.autopsy.directorytree.ExtractUnallocAction; import org.sleuthkit.autopsy.directorytree.FileSystemDetailsAction; -import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.mainui.datamodel.FileSystemContentSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.FileSystemColumnUtils; import org.sleuthkit.autopsy.mainui.datamodel.MediaTypeUtils; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java index c4ddbf366e..85c88c2a40 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.mainui.nodes; -import java.beans.PropertyChangeEvent; import java.util.Collection; import java.util.Comparator; import java.util.concurrent.ExecutionException; @@ -26,7 +25,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.openide.nodes.Children; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; -import org.sleuthkit.autopsy.ingest.ModuleContentEvent; import org.sleuthkit.autopsy.mainui.datamodel.FileExtDocumentFilter; import org.sleuthkit.autopsy.mainui.datamodel.FileExtExecutableFilter; import org.sleuthkit.autopsy.mainui.datamodel.FileExtRootFilter; @@ -37,7 +35,6 @@ import org.sleuthkit.autopsy.mainui.datamodel.FileTypeSizeSearchParams; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; -import org.sleuthkit.datamodel.AbstractFile; /** * From f9b59d066743da9d80bd650eedb13c87589c0fd1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 30 Nov 2021 20:11:04 -0500 Subject: [PATCH 02/24] type id updates --- .../autopsy/mainui/datamodel/AnalysisResultDAO.java | 8 ++++---- .../mainui/datamodel/AnalysisResultSearchParam.java | 2 +- .../autopsy/mainui/datamodel/CommAccountsDAO.java | 2 +- .../autopsy/mainui/datamodel/DataArtifactDAO.java | 2 +- .../autopsy/mainui/datamodel/DataArtifactSearchParam.java | 2 +- .../sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java | 6 +++--- .../org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java | 6 +++--- .../autopsy/mainui/nodes/AnalysisResultTypeFactory.java | 4 ++-- .../autopsy/mainui/nodes/DataArtifactTypeFactory.java | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java index 2a5e03e103..864e924acb 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java @@ -375,7 +375,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { private TreeItemDTO getTreeItem(BlackboardArtifact.Type type, Long dataSourceId, TreeDisplayCount displayCount) { return new TreeItemDTO<>( - BlackboardArtifact.Category.ANALYSIS_RESULT.name(), + AnalysisResultSearchParam.getTypeId(), new AnalysisResultSearchParam(type, dataSourceId), type.getTypeID(), type.getDisplayName(), @@ -478,7 +478,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { Long dataSourceId, String setName, String displayName, TreeDisplayCount displayCount) { return new TreeItemDTO<>( - type.getTypeName(), + AnalysisResultSetSearchParam.getTypeId(), new AnalysisResultSetSearchParam(type, dataSourceId, setName), setName == null ? 0 : setName, displayName, @@ -617,7 +617,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { } TreeItemDTO treeItem = new TreeItemDTO<>( - "KEYWORD_SEARCH_TERMS", + KeywordSearchTermParams.getTypeId(), new KeywordSearchTermParams(setName, searchTerm, searchType, hasChildren, dataSourceId), searchTermModified, searchTermModified, @@ -709,7 +709,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { long count = resultSet.getLong("count"); items.add(new TreeItemDTO<>( - "KEYWORD_MATCH", + KeywordMatchParams.getTypeId(), new KeywordMatchParams(setName, regexStr, keyword, searchType, dataSourceId), keyword, keyword == null ? "" : keyword, diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java index aca06e0081..c9ad697a82 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; */ public class AnalysisResultSearchParam extends BlackboardArtifactSearchParam { - private static final String TYPE_ID = "ANALYSIS_RESULT"; + private static final String TYPE_ID = BlackboardArtifact.Category.ANALYSIS_RESULT.name(); /** * @return The type id for this search parameter. diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java index 2b3e3e3dcb..9bc5d9d70e 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java @@ -162,7 +162,7 @@ public class CommAccountsDAO extends AbstractDAO { private static TreeResultsDTO.TreeItemDTO createAccountTreeItem(Account.Type accountType, Long dataSourceId, TreeResultsDTO.TreeDisplayCount count) { return new TreeResultsDTO.TreeItemDTO<>( - "ACCOUNTS", + CommAccountsSearchParams.getTypeId(), new CommAccountsSearchParams(accountType, dataSourceId), accountType.getTypeName(), accountType.getDisplayName(), diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java index 076f430992..2f7b22c400 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java @@ -228,7 +228,7 @@ public class DataArtifactDAO extends BlackboardArtifactDAO { private TreeItemDTO createDataArtifactTreeItem(BlackboardArtifact.Type artifactType, Long dataSourceId, TreeDisplayCount displayCount) { return new TreeResultsDTO.TreeItemDTO<>( - BlackboardArtifact.Category.DATA_ARTIFACT.name(), + DataArtifactSearchParam.getTypeId(), new DataArtifactSearchParam(artifactType, dataSourceId), artifactType.getTypeID(), artifactType.getDisplayName(), diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java index cf06b9a356..9fff59cd28 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; */ public class DataArtifactSearchParam extends BlackboardArtifactSearchParam { - private static final String TYPE_ID = "DATA_ARTIFACT"; + private static final String TYPE_ID = BlackboardArtifact.Category.DATA_ARTIFACT.name(); /** * @return The type id for this search parameter. diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java index de174d3477..c3c9840bae 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java @@ -472,7 +472,7 @@ public class FileSystemDAO extends AbstractDAO { List> treeItemRows = new ArrayList<>(); for (DataSource ds : Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getDataSourcesForHost(host)) { treeItemRows.add(new TreeResultsDTO.TreeItemDTO<>( - ds.getClass().getSimpleName(), + FileSystemContentSearchParam.getTypeId(), new FileSystemContentSearchParam(ds.getId()), ds, ds.getName(), @@ -499,7 +499,7 @@ public class FileSystemDAO extends AbstractDAO { List> treeItemRows = new ArrayList<>(); DataSource ds = Case.getCurrentCaseThrows().getSleuthkitCase().getDataSource(dataSourceObjId); treeItemRows.add(new TreeResultsDTO.TreeItemDTO<>( - ds.getClass().getSimpleName(), + FileSystemContentSearchParam.getTypeId(), new FileSystemContentSearchParam(ds.getId()), ds, ds.getName(), @@ -534,7 +534,7 @@ public class FileSystemDAO extends AbstractDAO { countForNode = getContentForTable(new FileSystemContentSearchParam(child.getId()), 0, null).getTotalResultsCount(); } treeItemRows.add(new TreeResultsDTO.TreeItemDTO<>( - child.getClass().getSimpleName(), + FileSystemContentSearchParam.getTypeId(), new FileSystemContentSearchParam(child.getId()), child, getNameForContent(child), diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index 1070fd9907..74190f7a01 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -316,7 +316,7 @@ public class ViewsDAO extends AbstractDAO { List> treeList = countsByFilter.entrySet().stream() .map(entry -> { return new TreeItemDTO<>( - "FILE_EXT", + FileTypeExtensionsSearchParams.getTypeId(), new FileTypeExtensionsSearchParams(entry.getKey(), dataSourceId), entry.getKey(), entry.getKey().getDisplayName(), @@ -350,7 +350,7 @@ public class ViewsDAO extends AbstractDAO { List> treeList = countsByFilter.entrySet().stream() .map(entry -> { return new TreeItemDTO<>( - "FILE_SIZE", + FileTypeSizeSearchParams.getTypeId(), new FileTypeSizeSearchParams(entry.getKey(), dataSourceId), entry.getKey(), entry.getKey().getDisplayName(), @@ -435,7 +435,7 @@ public class ViewsDAO extends AbstractDAO { : entry.getKey(); return new TreeItemDTO<>( - "FILE_MIME_TYPE", + FileTypeMimeSearchParams.getTypeId(), new FileTypeMimeSearchParams(entry.getKey(), dataSourceId), name, name, diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java index 6de6ec6752..07134c0bdb 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java @@ -107,7 +107,7 @@ public class AnalysisResultTypeFactory extends TreeChildFactory( - BlackboardArtifact.Category.ANALYSIS_RESULT.name(), + AnalysisResultSearchParam.getTypeId(), new AnalysisResultSearchParam(searchParam.getArtifactType(), searchParam.getDataSourceId()), searchParam.getArtifactType().getTypeID(), searchParam.getArtifactType().getDisplayName(), @@ -208,7 +208,7 @@ public class AnalysisResultTypeFactory extends TreeChildFactory( - TBD, + AnalysisResultSetSearchParam.getTypeId(), new AnalysisResultSetSearchParam(searchParam.getArtifactType(), searchParam.getDataSourceId(), searchParam.getSetName()), searchParam.getArtifactType().getTypeID(), searchParam.getArtifactType().getDisplayName(), diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java index 56b56979a7..1812c37c31 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java @@ -76,7 +76,7 @@ public class DataArtifactTypeFactory extends TreeChildFactory( - BlackboardArtifact.Category.DATA_ARTIFACT.name(), + DataArtifactSearchParam.getTypeId(), new DataArtifactSearchParam(searchParam.getArtifactType(), searchParam.getDataSourceId()), searchParam.getArtifactType().getTypeID(), searchParam.getArtifactType().getDisplayName(), From a00302957db1719338236badb1364ef8753b9cf5 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 1 Dec 2021 11:32:27 -0500 Subject: [PATCH 03/24] updates to analysis results factory --- .../nodes/AnalysisResultTypeFactory.java | 64 +++++++++++++++---- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java index 07134c0bdb..180a7ed193 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java @@ -33,8 +33,6 @@ import org.sleuthkit.autopsy.datamodel.utils.IconsUtil; import org.sleuthkit.autopsy.mainui.datamodel.AnalysisResultDAO; import org.sleuthkit.autopsy.mainui.datamodel.AnalysisResultSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.AnalysisResultSetSearchParam; -import org.sleuthkit.autopsy.mainui.datamodel.DataArtifactDAO; -import org.sleuthkit.autopsy.mainui.datamodel.DataArtifactSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.KeywordHitSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; @@ -108,7 +106,7 @@ public class AnalysisResultTypeFactory extends TreeChildFactory( AnalysisResultSearchParam.getTypeId(), - new AnalysisResultSearchParam(searchParam.getArtifactType(), searchParam.getDataSourceId()), + new AnalysisResultSearchParam(searchParam.getArtifactType(), this.dataSourceId), searchParam.getArtifactType().getTypeID(), searchParam.getArtifactType().getDisplayName(), originalTreeItem.getDisplayCount()); @@ -202,16 +200,16 @@ public class AnalysisResultTypeFactory extends TreeChildFactory originalTreeItem = super.getTypedTreeItem(treeEvt, AnalysisResultSetSearchParam.class); if (originalTreeItem != null - && !AnalysisResultDAO.getIgnoredTreeTypes().contains(originalTreeItem.getSearchParams().getArtifactType()) + && originalTreeItem.getSearchParams().getArtifactType().equals(this.artifactType) && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. AnalysisResultSetSearchParam searchParam = originalTreeItem.getSearchParams(); return new TreeResultsDTO.TreeItemDTO<>( AnalysisResultSetSearchParam.getTypeId(), - new AnalysisResultSetSearchParam(searchParam.getArtifactType(), searchParam.getDataSourceId(), searchParam.getSetName()), - searchParam.getArtifactType().getTypeID(), - searchParam.getArtifactType().getDisplayName(), + new AnalysisResultSetSearchParam(this.artifactType, this.dataSourceId, searchParam.getSetName()), + searchParam.getSetName(), + searchParam.getSetName() == null ? nullSetName : searchParam.getSetName(), originalTreeItem.getDisplayCount()); } return null; @@ -309,8 +307,30 @@ public class AnalysisResultTypeFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent daoEvt) { - // GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, KeywordSearchTermParams.class); + + if (originalTreeItem != null + && Objects.equals(originalTreeItem.getSearchParams().getSetName(), this.setParams.getSetName()) + && (this.setParams.getDataSourceId() == null + || Objects.equals(this.setParams.getDataSourceId(), originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + KeywordSearchTermParams searchParam = originalTreeItem.getSearchParams(); + return new TreeResultsDTO.TreeItemDTO<>( + KeywordSearchTermParams.getTypeId(), + new KeywordSearchTermParams( + this.setParams.getSetName(), + searchParam.getSearchTerm(), + searchParam.getSearchType(), + searchParam.hasChildren(), + this.setParams.getDataSourceId() + ), + nameTBD, + displayNameTBD, + originalTreeItem.getDisplayCount() + ); + } return null; } @@ -389,8 +409,30 @@ public class AnalysisResultTypeFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent daoEvt) { - // GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, KeywordMatchParams.class); + + if (originalTreeItem != null + && Objects.equals(originalTreeItem.getSearchParams().getSetName(), this.setParams.getSetName()) + && (this.setParams.getDataSourceId() == null + || Objects.equals(this.setParams.getDataSourceId(), originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + KeywordMatchParams searchParam = originalTreeItem.getSearchParams(); + return new TreeResultsDTO.TreeItemDTO<>( + KeywordMatchParams.getTypeId(), + new KeywordMatchParams( + this.setParams.getSetName(), + this.setParams.getSearchTerm(), + searchParam.getKeywordMatch(), + this.setParams.getSearchType(), + this.setParams.getDataSourceId() + ), + searchParam.getKeywordMatch(), + searchParam.getKeywordMatch() == null ? "" : searchParam.getKeywordMatch(), + originalTreeItem.getDisplayCount() + ); + } return null; } From 45c07470a97202c467e673eb03ac75bbd156c545 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 1 Dec 2021 13:28:53 -0500 Subject: [PATCH 04/24] updates for getOrCreateRelevantChild --- .../mainui/datamodel/AnalysisResultDAO.java | 43 +++++++---- .../nodes/AnalysisResultTypeFactory.java | 8 +- .../mainui/nodes/ViewsTypeFactory.java | 74 +++++++++++++++++-- 3 files changed, 98 insertions(+), 27 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java index 864e924acb..736c4aec04 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java @@ -599,22 +599,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { long count = resultSet.getLong("count"); boolean hasChildren = resultSet.getBoolean("has_children"); - String searchTermModified; - switch (searchType) { - case 0: - searchTermModified = Bundle.AnalysisResultDAO_getKeywordSearchTermCounts_exactMatch(searchTerm == null ? "" : searchTerm); - break; - case 1: - searchTermModified = Bundle.AnalysisResultDAO_getKeywordSearchTermCounts_substringMatch(searchTerm == null ? "" : searchTerm); - break; - case 2: - searchTermModified = Bundle.AnalysisResultDAO_getKeywordSearchTermCounts_regexMatch(searchTerm == null ? "" : searchTerm); - break; - default: - logger.log(Level.WARNING, MessageFormat.format("Non-standard search type value: {0}.", searchType)); - searchTermModified = searchTerm; - break; - } + String searchTermModified = getSearchTermDisplayName(searchTerm, searchType); TreeItemDTO treeItem = new TreeItemDTO<>( KeywordSearchTermParams.getTypeId(), @@ -638,6 +623,32 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { } } + /** + * Returns the UI display name for a search term. + * @param searchTerm The search term. + * @param searchType The search type enum value. + * @return The display name. + */ + public String getSearchTermDisplayName(String searchTerm, int searchType) { + String searchTermModified; + switch (searchType) { + case 0: + searchTermModified = Bundle.AnalysisResultDAO_getKeywordSearchTermCounts_exactMatch(searchTerm == null ? "" : searchTerm); + break; + case 1: + searchTermModified = Bundle.AnalysisResultDAO_getKeywordSearchTermCounts_substringMatch(searchTerm == null ? "" : searchTerm); + break; + case 2: + searchTermModified = Bundle.AnalysisResultDAO_getKeywordSearchTermCounts_regexMatch(searchTerm == null ? "" : searchTerm); + break; + default: + logger.log(Level.WARNING, MessageFormat.format("Non-standard search type value: {0}.", searchType)); + searchTermModified = searchTerm; + break; + } + return searchTermModified; + } + /** * Get counts for string matches of a particular regex/substring search * term. diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java index 180a7ed193..853eb86556 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java @@ -315,8 +315,10 @@ public class AnalysisResultTypeFactory extends TreeChildFactory( KeywordSearchTermParams.getTypeId(), new KeywordSearchTermParams( @@ -326,8 +328,8 @@ public class AnalysisResultTypeFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent daoEvt) { - // GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, FileTypeSizeSearchParams.class); + + if (originalTreeItem != null + && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + FileTypeSizeSearchParams searchParam = originalTreeItem.getSearchParams(); + return new TreeResultsDTO.TreeItemDTO<>( + AnalysisResultSearchParam.getTypeId(), + new FileTypeSizeSearchParams(searchParam.getSizeFilter(), this.dataSourceId), + searchParam.getSizeFilter(), + searchParam.getSizeFilter().getDisplayName(), + originalTreeItem.getDisplayCount()); + } return null; } @@ -130,9 +145,23 @@ public class ViewsTypeFactory { } @Override - protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent daoEvt) { - // GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, FileTypeMimeSearchParams.class); + + if (originalTreeItem != null + && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + FileTypeMimeSearchParams searchParam = originalTreeItem.getSearchParams(); + return new TreeResultsDTO.TreeItemDTO<>( + AnalysisResultSearchParam.getTypeId(), + new FileTypeMimeSearchParams(searchParam.getMimeType(), this.dataSourceId), + searchParam.getMimeType(), + searchParam.getMimeType(), + originalTreeItem.getDisplayCount()); + } return null; + } @Override @@ -190,8 +219,24 @@ public class ViewsTypeFactory { } @Override - protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent daoEvt) { - // GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, FileTypeMimeSearchParams.class); + + String prefixWithSlash = this.mimeTypePrefix + "/"; + if (originalTreeItem != null + && (originalTreeItem.getSearchParams().getMimeType().startsWith(prefixWithSlash)) + && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + FileTypeMimeSearchParams searchParam = originalTreeItem.getSearchParams(); + String mimeSuffix = searchParam.getMimeType().substring(prefixWithSlash.length()); + return new TreeResultsDTO.TreeItemDTO<>( + AnalysisResultSearchParam.getTypeId(), + new FileTypeMimeSearchParams(searchParam.getMimeType(), this.dataSourceId), + mimeSuffix, + mimeSuffix, + originalTreeItem.getDisplayCount()); + } return null; } @@ -275,8 +320,21 @@ public class ViewsTypeFactory { } @Override - protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent daoEvt) { - //GVDTODO + protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { + TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, FileTypeExtensionsSearchParams.class); + + if (originalTreeItem != null + && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { + + // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. + FileTypeExtensionsSearchParams searchParam = originalTreeItem.getSearchParams(); + return new TreeResultsDTO.TreeItemDTO<>( + AnalysisResultSearchParam.getTypeId(), + new FileTypeExtensionsSearchParams(searchParam.getFilter(), this.dataSourceId), + searchParam.getFilter(), + searchParam.getFilter().getDisplayName(), + originalTreeItem.getDisplayCount()); + } return null; } From 32fe91977fbdf4bb05164f43825c2db987bf36f4 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 1 Dec 2021 14:06:23 -0500 Subject: [PATCH 05/24] updates --- .../autopsy/mainui/datamodel/AbstractDAO.java | 10 +++--- .../mainui/datamodel/AnalysisResultDAO.java | 32 ++++++++----------- .../mainui/datamodel/CommAccountsDAO.java | 4 +-- .../mainui/datamodel/DataArtifactDAO.java | 4 +-- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java index 9172770c47..93f8fb7f03 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java @@ -26,9 +26,11 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; +import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeDisplayCount; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeCounts; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; @@ -121,9 +123,9 @@ abstract class AbstractDAO { * @param converter The means of acquiring a tree item dto to be placed in the TreeEvent. * @return The generated tree events. */ - static Set getIngestCompleteEvents(TreeCounts treeCounts, Function> converter) { + static Set getIngestCompleteEvents(TreeCounts treeCounts, BiFunction> converter) { return treeCounts.flushEvents().stream() - .map(daoEvt -> new TreeEvent(converter.apply(daoEvt), true)) + .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) .collect(Collectors.toSet()); } @@ -133,9 +135,9 @@ abstract class AbstractDAO { * @param converter The means of acquiring a tree item dto to be placed in the TreeEvent. * @return The generated tree events. */ - static Set getRefreshEvents(TreeCounts treeCounts, Function> converter) { + static Set getRefreshEvents(TreeCounts treeCounts, BiFunction> converter) { return treeCounts.getEventTimeouts().stream() - .map(daoEvt -> new TreeEvent(converter.apply(daoEvt), true)) + .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) .collect(Collectors.toSet()); } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java index 736c4aec04..10965a2c2a 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java @@ -625,8 +625,10 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { /** * Returns the UI display name for a search term. + * * @param searchTerm The search term. * @param searchType The search type enum value. + * * @return The display name. */ public String getSearchTermDisplayName(String searchTerm, int searchType) { @@ -783,7 +785,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { List daoEvents = getResultViewEvents(analysisResultMap, setMap); Collection treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() - .map(arEvt -> getTreeEvent(arEvt, false)) + .map(arEvt -> new TreeEvent(getTreeItem(arEvt, TreeDisplayCount.INDETERMINATE), false)) .collect(Collectors.toList()); return Stream.of(daoEvents, treeEvents) @@ -852,42 +854,34 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { } /** - * Creates a TreeEvent instance based on the analysis result event and + * Creates a TreeItemDTO instance based on the analysis result event and * whether or not this event should trigger a full refresh of counts. * - * @param arEvt The analysis result event. - * @param shouldRefresh Whether or not this tree event should trigger a full - * refresh of counts. + * @param arEvt The analysis result event. + * @param displayCount The count to display. * * @return The tree event. */ - private TreeEvent getTreeEvent(AnalysisResultEvent arEvt, boolean shouldRefresh) { + private TreeItemDTO getTreeItem(AnalysisResultEvent arEvt, TreeDisplayCount displayCount) { // GVDTODO handle keyword items when integrated if (arEvt instanceof AnalysisResultSetEvent) { AnalysisResultSetEvent setEvt = (AnalysisResultSetEvent) arEvt; - return new TreeEvent(getSetTreeItem(setEvt.getArtifactType(), setEvt.getDataSourceId(), + return getSetTreeItem(setEvt.getArtifactType(), setEvt.getDataSourceId(), setEvt.getSetName(), setEvt.getSetName() == null ? "" : setEvt.getSetName(), - shouldRefresh ? TreeDisplayCount.UNSPECIFIED : TreeDisplayCount.INDETERMINATE), - shouldRefresh); + displayCount); } else { - return new TreeEvent(getTreeItem(arEvt.getArtifactType(), arEvt.getDataSourceId(), - shouldRefresh ? TreeDisplayCount.UNSPECIFIED : TreeDisplayCount.INDETERMINATE), - shouldRefresh); + return getTreeItem(arEvt.getArtifactType(), arEvt.getDataSourceId(), displayCount); } } @Override - Set handleIngestComplete() { - return this.treeCounts.flushEvents().stream() - .map(arEvt -> getTreeEvent(arEvt, true)) - .collect(Collectors.toSet()); + Set handleIngestComplete() { + return getIngestCompleteEvents(this.treeCounts, (arEvt, count) -> getTreeItem(arEvt, count)); } @Override Set shouldRefreshTree() { - return this.treeCounts.getEventTimeouts().stream() - .map(arEvt -> getTreeEvent(arEvt, true)) - .collect(Collectors.toSet()); + return getRefreshEvents(this.treeCounts, (arEvt, count) -> getTreeItem(arEvt, count)); } /** diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java index 9bc5d9d70e..d9de9f4cb7 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java @@ -238,7 +238,7 @@ public class CommAccountsDAO extends AbstractDAO { Set handleIngestComplete() { return getIngestCompleteEvents( this.accountCounts, - (daoEvt) -> createAccountTreeItem(daoEvt.getAccountType(), daoEvt.getDataSourceId(), TreeResultsDTO.TreeDisplayCount.UNSPECIFIED) + (daoEvt, count) -> createAccountTreeItem(daoEvt.getAccountType(), daoEvt.getDataSourceId(), count) ); } @@ -246,7 +246,7 @@ public class CommAccountsDAO extends AbstractDAO { Set shouldRefreshTree() { return getRefreshEvents( this.accountCounts, - (daoEvt) -> createAccountTreeItem(daoEvt.getAccountType(), daoEvt.getDataSourceId(), TreeResultsDTO.TreeDisplayCount.UNSPECIFIED) + (daoEvt, count) -> createAccountTreeItem(daoEvt.getAccountType(), daoEvt.getDataSourceId(), count) ); } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java index 071ebc46dd..e3a3ab8486 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java @@ -255,13 +255,13 @@ public class DataArtifactDAO extends BlackboardArtifactDAO { @Override Set handleIngestComplete() { return getIngestCompleteEvents(this.treeCounts, - (daoEvt) -> createDataArtifactTreeItem(daoEvt.getArtifactType(), daoEvt.getDataSourceId(), TreeDisplayCount.UNSPECIFIED)); + (daoEvt, count) -> createDataArtifactTreeItem(daoEvt.getArtifactType(), daoEvt.getDataSourceId(), count)); } @Override Set shouldRefreshTree() { return getRefreshEvents(this.treeCounts, - (daoEvt) -> createDataArtifactTreeItem(daoEvt.getArtifactType(), daoEvt.getDataSourceId(), TreeDisplayCount.UNSPECIFIED)); + (daoEvt, count) -> createDataArtifactTreeItem(daoEvt.getArtifactType(), daoEvt.getDataSourceId(), count)); } From e3229354e5329a28ab5cb94e3800592df1a4b99c Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 1 Dec 2021 19:28:09 -0500 Subject: [PATCH 06/24] analysis result DAO changes --- .../mainui/datamodel/AnalysisResultDAO.java | 143 ++++++++++-------- .../datamodel/events/KeywordHitEvent.java | 26 +++- 2 files changed, 104 insertions(+), 65 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java index 10965a2c2a..fd98ac1ed4 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java @@ -51,6 +51,7 @@ import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeDisplayCount; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.DAOEventUtils; +import org.sleuthkit.autopsy.mainui.datamodel.events.KeywordHitEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeCounts; import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; @@ -333,18 +334,6 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { return keywordHitCache.get(searchParams, () -> fetchSetNameHitsForTable(searchParams)); } - public void dropAnalysisResultCache() { - analysisResultCache.invalidateAll(); - } - - public void dropHashHitCache() { - setHitCache.invalidateAll(); - } - - public void dropKeywordHitCache() { - keywordHitCache.invalidateAll(); - } - /** * Returns a search results dto containing rows of counts data. * @@ -358,10 +347,22 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { */ public TreeResultsDTO getAnalysisResultCounts(Long dataSourceId) throws ExecutionException { try { + + Set indeterminateTypes = this.treeCounts.getEnqueued().stream() + .filter(evt -> dataSourceId == null || Objects.equals(evt.getDataSourceId(), dataSourceId)) + .map(evt -> evt.getArtifactType()) + .collect(Collectors.toSet()); + // get row dto's sorted by display name Map typeCounts = getCounts(BlackboardArtifact.Category.ANALYSIS_RESULT, dataSourceId); List> treeItemRows = typeCounts.entrySet().stream() - .map(entry -> getTreeItem(entry.getKey(), dataSourceId, TreeDisplayCount.getDeterminate(entry.getValue()))) + .map(entry -> { + TreeDisplayCount displayCount = indeterminateTypes.contains(entry.getKey()) + ? TreeDisplayCount.INDETERMINATE + : TreeDisplayCount.getDeterminate(entry.getValue()); + + return getTreeItem(entry.getKey(), dataSourceId, displayCount); + }) .sorted(Comparator.comparing(countRow -> countRow.getDisplayName())) .collect(Collectors.toList()); @@ -458,16 +459,29 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { Long dataSourceId, String nullSetName) throws IllegalArgumentException, ExecutionException { + Set indeterminateSetNames = new HashSet<>(); + for (AnalysisResultEvent evt : this.treeCounts.getEnqueued()) { + if (evt instanceof AnalysisResultSetEvent + && (dataSourceId == null || Objects.equals(evt.getDataSourceId(), dataSourceId)) + && evt.getArtifactType().equals(type)) { + indeterminateSetNames.add(((AnalysisResultSetEvent) evt).getSetName()); + } + } + List> allSets = getSetCountsMap(type, BlackboardAttribute.Type.TSK_SET_NAME, dataSourceId).entrySet().stream() .filter(entry -> nullSetName != null || entry.getKey() != null) .sorted((a, b) -> compareSetStrings(a.getKey(), b.getKey())) .map(entry -> { + TreeDisplayCount displayCount = indeterminateSetNames.contains(entry.getKey()) + ? TreeDisplayCount.INDETERMINATE + : TreeDisplayCount.getDeterminate(entry.getValue()); + return getSetTreeItem(type, dataSourceId, entry.getKey(), entry.getKey() == null ? nullSetName : entry.getKey(), - TreeDisplayCount.getDeterminate(entry.getValue())); + displayCount); }) .collect(Collectors.toList()); @@ -528,6 +542,18 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { throw new IllegalArgumentException("Expected data source id to be > 0"); } + Set> indeterminateSearchTerms = new HashSet<>(); + for (AnalysisResultEvent evt : this.treeCounts.getEnqueued()) { + if (evt instanceof KeywordHitEvent + && (dataSourceId == null || Objects.equals(evt.getDataSourceId(), dataSourceId)) + && evt.getArtifactType().equals(BlackboardArtifact.Type.TSK_KEYWORD_HIT) + && Objects.equals(((KeywordHitEvent) evt).getSetName(), setName)) { + + KeywordHitEvent keywordEvt = (KeywordHitEvent) evt; + indeterminateSearchTerms.add(Pair.of(keywordEvt.getSearchString(), keywordEvt.getSearchType())); + } + } + String dataSourceClause = dataSourceId == null ? "" : "AND art.data_source_obj_id = ?\n"; @@ -601,12 +627,16 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { String searchTermModified = getSearchTermDisplayName(searchTerm, searchType); + TreeDisplayCount displayCount = indeterminateSearchTerms.contains(Pair.of(searchTerm, searchType)) + ? TreeDisplayCount.INDETERMINATE + : TreeDisplayCount.getDeterminate(count); + TreeItemDTO treeItem = new TreeItemDTO<>( KeywordSearchTermParams.getTypeId(), new KeywordSearchTermParams(setName, searchTerm, searchType, hasChildren, dataSourceId), searchTermModified, searchTermModified, - TreeDisplayCount.getDeterminate(count) + displayCount ); items.add(treeItem); @@ -700,6 +730,23 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { + "AND res.search_type = ?\n" + "GROUP BY keyword"; + Set indeterminateMatches = new HashSet<>(); + for (AnalysisResultEvent evt : this.treeCounts.getEnqueued()) { + if (evt instanceof KeywordHitEvent + && (dataSourceId == null || Objects.equals(evt.getDataSourceId(), dataSourceId)) + && evt.getArtifactType().equals(BlackboardArtifact.Type.TSK_KEYWORD_HIT)) { + + KeywordHitEvent keywordEvt = (KeywordHitEvent) evt; + if (Objects.equals(keywordEvt.getSetName(), setName) + && Objects.equals(keywordEvt.getSearchString(), regexStr) + && keywordEvt.getSearchType() == searchType) { + + indeterminateMatches.add(keywordEvt.getMatch()); + } + + } + } + try (CaseDbPreparedStatement preparedStatement = getCase().getCaseDbAccessManager().prepareSelect(query)) { // get artifact types and counts int paramIdx = 0; @@ -721,12 +768,17 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { String keyword = resultSet.getString("keyword"); long count = resultSet.getLong("count"); + TreeDisplayCount displayCount = indeterminateMatches.contains(keyword) + ? TreeDisplayCount.INDETERMINATE + : TreeDisplayCount.getDeterminate(count); + items.add(new TreeItemDTO<>( KeywordMatchParams.getTypeId(), new KeywordMatchParams(setName, regexStr, keyword, searchType, dataSourceId), keyword, keyword == null ? "" : keyword, - TreeDisplayCount.getDeterminate(count))); + displayCount + )); } } catch (SQLException ex) { logger.log(Level.WARNING, "An error occurred while fetching results from result set.", ex); @@ -748,7 +800,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { } @Override - Set processEvent(PropertyChangeEvent evt) { + Set processEvent(PropertyChangeEvent evt) { // get a grouping of artifacts mapping the artifact type id to data source id. Map> analysisResultMap = new HashMap<>(); Map, Set> setMap = new HashMap<>(); @@ -781,16 +833,11 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { return Collections.emptySet(); } - clearRelevantCacheEntries(analysisResultMap, setMap); + invalidateKeys(this.analysisResultCache, ar -> Pair.of(ar.getArtifactType(), ar.getDataSourceId()), analysisResultMap); + invalidateKeys(this.setHitCache, ar -> Pair.of(Pair.of(ar.getArtifactType(), ar.getSetName()), ar.getDataSourceId()), setMap); - List daoEvents = getResultViewEvents(analysisResultMap, setMap); - Collection treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() - .map(arEvt -> new TreeEvent(getTreeItem(arEvt, TreeDisplayCount.INDETERMINATE), false)) - .collect(Collectors.toList()); - - return Stream.of(daoEvents, treeEvents) - .flatMap(lst -> lst.stream()) - .collect(Collectors.toSet()); + // GVDTODO handle keyword hits + return getResultViewEvents(analysisResultMap, setMap); } /** @@ -806,7 +853,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { * * @return The list of dao events. */ - private List getResultViewEvents(Map> analysisResultMap, Map, Set> resultsWithSetMap) { + private Set getResultViewEvents(Map> analysisResultMap, Map, Set> resultsWithSetMap) { Stream analysisResultEvts = analysisResultMap.entrySet().stream() .flatMap(entry -> entry.getValue().stream().map(dsId -> new AnalysisResultEvent(entry.getKey(), dsId))); @@ -814,43 +861,18 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { .flatMap(entry -> entry.getValue().stream().map(dsId -> new AnalysisResultSetEvent(entry.getKey().getRight(), entry.getKey().getLeft(), dsId))); // GVDTODO handle keyword hits - return Stream.of(analysisResultEvts, analysisResultSetEvts) + + List daoEvents = Stream.of(analysisResultEvts, analysisResultSetEvts) .flatMap(s -> s) .collect(Collectors.toList()); - } - /** - * Clears cache entries given the provided digests of autopsy events. - * - * @param analysisResultMap Contains the analysis results that do not use a - * set name. A mapping of analysis result type ids - * to data sources where the results were created. - * @param resultsWithSetMap Contains the anlaysis results that do use a set - * name. A mapping of (analysis result type id, set - * name) to data sources where results were - * created. - */ - private void clearRelevantCacheEntries(Map> analysisResultMap, Map, Set> resultsWithSetMap) { - ConcurrentMap, AnalysisResultTableSearchResultsDTO> arConcurrentMap = this.analysisResultCache.asMap(); - arConcurrentMap.forEach((k, v) -> { - BlackboardArtifactSearchParam searchParam = k.getParamData(); - Set dsIds = analysisResultMap.get(searchParam.getArtifactType()); - if (dsIds != null && (searchParam.getDataSourceId() == null || dsIds.contains(searchParam.getDataSourceId()))) { - arConcurrentMap.remove(k); - } - }); + Collection treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() + .map(arEvt -> new TreeEvent(getTreeItem(arEvt, TreeDisplayCount.INDETERMINATE), false)) + .collect(Collectors.toList()); - ConcurrentMap, AnalysisResultTableSearchResultsDTO> setConcurrentMap = this.setHitCache.asMap(); - setConcurrentMap.forEach((k, v) -> { - AnalysisResultSetSearchParam searchParam = k.getParamData(); - Set dsIds = resultsWithSetMap.get(Pair.of(searchParam.getArtifactType(), searchParam.getSetName())); - if (dsIds != null && (searchParam.getDataSourceId() == null || dsIds.contains(searchParam.getDataSourceId()))) { - arConcurrentMap.remove(k); - } - }); - - // GVDTODO handle clearing cache for keyword search hits - // private final Cache, AnalysisResultTableSearchResultsDTO> keywordHitCache = CacheBuilder.newBuilder().maximumSize(1000).build(); + return Stream.of(daoEvents, treeEvents) + .flatMap(lst -> lst.stream()) + .collect(Collectors.toSet()); } /** @@ -882,6 +904,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { @Override Set shouldRefreshTree() { return getRefreshEvents(this.treeCounts, (arEvt, count) -> getTreeItem(arEvt, count)); + } /** diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/KeywordHitEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/KeywordHitEvent.java index 6141ac7805..eac35fa219 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/KeywordHitEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/KeywordHitEvent.java @@ -26,20 +26,36 @@ import org.sleuthkit.datamodel.BlackboardArtifact; */ public class KeywordHitEvent extends AnalysisResultSetEvent { - private final String regex; + private final String searchString; private final String match; + private final int searchType; - public KeywordHitEvent(String regex, String match, String setName, BlackboardArtifact.Type artifactType, long dataSourceId) { + /** + * Main constructor. + * + * @param searchString The search string or regex. + * @param match The match string. + * @param searchType THe search type. + * @param setName The set name. + * @param artifactType The artifact type. + * @param dataSourceId The data source id. + */ + public KeywordHitEvent(String searchString, String match, int searchType, String setName, BlackboardArtifact.Type artifactType, long dataSourceId) { super(setName, artifactType, dataSourceId); - this.regex = regex; + this.searchString = searchString; this.match = match; + this.searchType = searchType; } - public String getRegex() { - return regex; + public String getSearchString() { + return searchString; } public String getMatch() { return match; } + + public int getSearchType() { + return searchType; + } } From 37574efca94fea22cf10f11c86dc4e240c987cd7 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 1 Dec 2021 19:52:30 -0500 Subject: [PATCH 07/24] views counts updates --- .../autopsy/mainui/datamodel/ViewsDAO.java | 63 +++++++++++++++++-- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index 74190f7a01..e40eca8495 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -53,6 +53,7 @@ import org.sleuthkit.autopsy.mainui.datamodel.events.DAOEventUtils; import org.sleuthkit.autopsy.mainui.datamodel.events.FileTypeExtensionsEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileTypeMimeEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileTypeSizeEvent; +import org.sleuthkit.autopsy.mainui.datamodel.events.TreeCounts; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; import org.sleuthkit.datamodel.AbstractFile; @@ -72,7 +73,6 @@ public class ViewsDAO extends AbstractDAO { private static final int CACHE_SIZE = 15; // rule of thumb: 5 entries times number of cached SearchParams sub-types private static final long CACHE_DURATION = 2; private static final TimeUnit CACHE_DURATION_UNITS = TimeUnit.MINUTES; - private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); private static final String FILE_VIEW_EXT_TYPE_ID = "FILE_VIEW_BY_EXT"; @@ -86,6 +86,9 @@ public class ViewsDAO extends AbstractDAO { return instance; } + private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); + private final TreeCounts treeCounts = new TreeCounts<>(); + private SleuthkitCase getCase() throws NoCurrentCaseException { return Case.getCurrentCaseThrows().getSleuthkitCase(); } @@ -306,6 +309,20 @@ public class ViewsDAO extends AbstractDAO { * @throws ExecutionException */ public TreeResultsDTO getFileExtCounts(Collection filters, Long dataSourceId) throws IllegalArgumentException, ExecutionException { + Set indeterminateFilters = new HashSet<>(); + for (DAOEvent evt : this.treeCounts.getEnqueued()) { + if (evt instanceof FileTypeExtensionsEvent) { + FileTypeExtensionsEvent extEvt = (FileTypeExtensionsEvent) evt; + if (dataSourceId == null || Objects.equals(extEvt.getDataSourceId(), dataSourceId)) { + for (FileExtSearchFilter filter : filters) { + if (filter.getFilter().contains(evt)) { + indeterminateFilters.add(filter); + } + } + } + } + } + Map whereClauses = filters.stream() .collect(Collectors.toMap( filter -> filter, @@ -315,12 +332,16 @@ public class ViewsDAO extends AbstractDAO { List> treeList = countsByFilter.entrySet().stream() .map(entry -> { + TreeDisplayCount displayCount = indeterminateFilters.contains(entry.getKey()) + ? TreeDisplayCount.INDETERMINATE + : TreeDisplayCount.getDeterminate(entry.getValue()); + return new TreeItemDTO<>( FileTypeExtensionsSearchParams.getTypeId(), new FileTypeExtensionsSearchParams(entry.getKey(), dataSourceId), entry.getKey(), entry.getKey().getDisplayName(), - TreeDisplayCount.getDeterminate(entry.getValue())); + displayCount); }) .sorted((a, b) -> a.getDisplayName().compareToIgnoreCase(b.getDisplayName())) .collect(Collectors.toList()); @@ -340,6 +361,16 @@ public class ViewsDAO extends AbstractDAO { * @throws ExecutionException */ public TreeResultsDTO getFileSizeCounts(Long dataSourceId) throws IllegalArgumentException, ExecutionException { + Set indeterminateFilters = new HashSet<>(); + for (DAOEvent evt : this.treeCounts.getEnqueued()) { + if (evt instanceof FileTypeSizeEvent) { + FileTypeSizeEvent sizeEvt = (FileTypeSizeEvent) evt; + if (dataSourceId == null || Objects.equals(sizeEvt.getDataSourceId(), dataSourceId)) { + indeterminateFilters.add(sizeEvt.getSizeFilter()); + } + } + } + Map whereClauses = Stream.of(FileSizeFilter.values()) .collect(Collectors.toMap( filter -> filter, @@ -349,12 +380,16 @@ public class ViewsDAO extends AbstractDAO { List> treeList = countsByFilter.entrySet().stream() .map(entry -> { + TreeDisplayCount displayCount = indeterminateFilters.contains(entry.getKey()) + ? TreeDisplayCount.INDETERMINATE + : TreeDisplayCount.getDeterminate(entry.getValue()); + return new TreeItemDTO<>( FileTypeSizeSearchParams.getTypeId(), new FileTypeSizeSearchParams(entry.getKey(), dataSourceId), entry.getKey(), entry.getKey().getDisplayName(), - TreeDisplayCount.getDeterminate(entry.getValue())); + displayCount); }) .sorted((a, b) -> a.getDisplayName().compareToIgnoreCase(b.getDisplayName())) .collect(Collectors.toList()); @@ -379,6 +414,22 @@ public class ViewsDAO extends AbstractDAO { String prefixWithSlash = StringUtils.isNotBlank(prefix) ? prefix.replaceAll("/", "") + "/" : null; String likeItem = StringUtils.isNotBlank(prefixWithSlash) ? prefixWithSlash.replaceAll("%", "") + "%" : null; + Set indeterminateMimeTypes = new HashSet<>(); + for (DAOEvent evt : this.treeCounts.getEnqueued()) { + if (evt instanceof FileTypeMimeEvent) { + FileTypeMimeEvent mimeEvt = (FileTypeMimeEvent) evt; + if ((dataSourceId == null || Objects.equals(mimeEvt.getDataSourceId(), dataSourceId)) + && (prefixWithSlash == null || mimeEvt.getMimeType().startsWith(prefixWithSlash))) { + + String mimePortion = prefixWithSlash != null + ? mimeEvt.getMimeType().substring(prefixWithSlash.length()) + : mimeEvt.getMimeType().substring(0, mimeEvt.getMimeType().indexOf("/")); + + indeterminateMimeTypes.add(mimePortion); + } + } + } + String baseFilter = "WHERE " + getBaseFileMimeFilter() + getDataSourceAndClause(dataSourceId) + (StringUtils.isNotBlank(prefix) ? " AND mime_type LIKE ? " : " AND mime_type IS NOT NULL "); @@ -434,12 +485,16 @@ public class ViewsDAO extends AbstractDAO { ? entry.getKey().substring(prefixWithSlash.length()) : entry.getKey(); + TreeDisplayCount displayCount = indeterminateMimeTypes.contains(name) + ? TreeDisplayCount.INDETERMINATE + : TreeDisplayCount.getDeterminate(entry.getValue()); + return new TreeItemDTO<>( FileTypeMimeSearchParams.getTypeId(), new FileTypeMimeSearchParams(entry.getKey(), dataSourceId), name, name, - TreeDisplayCount.getDeterminate(entry.getValue())); + displayCount); }) .sorted((a, b) -> stringCompare(a.getSearchParams().getMimeType(), b.getSearchParams().getMimeType())) .collect(Collectors.toList()); From 8de4b5a4582f02c439f6bc8dc23002fc4d6a6e9f Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 2 Dec 2021 11:12:52 -0500 Subject: [PATCH 08/24] working through views event changes --- .../mainui/datamodel/TreeResultsDTO.java | 1 + .../autopsy/mainui/datamodel/ViewsDAO.java | 124 +++++++++++++----- .../events/FileTypeExtensionsEvent.java | 19 +-- .../datamodel/events/FileTypeMimeEvent.java | 30 +++-- 4 files changed, 127 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TreeResultsDTO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TreeResultsDTO.java index 81f176190d..e2eab8ff7e 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TreeResultsDTO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TreeResultsDTO.java @@ -87,6 +87,7 @@ public class TreeResultsDTO { case INDETERMINATE: return "..."; case NOT_SHOWN: + case UNSPECIFIED: default: return ""; } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index e40eca8495..90e8a9304d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -73,6 +73,14 @@ public class ViewsDAO extends AbstractDAO { private static final int CACHE_SIZE = 15; // rule of thumb: 5 entries times number of cached SearchParams sub-types private static final long CACHE_DURATION = 2; private static final TimeUnit CACHE_DURATION_UNITS = TimeUnit.MINUTES; + private static final Map> EXTENSION_FILTER_MAP + = Stream.of((FileExtSearchFilter[]) FileExtRootFilter.values(), FileExtDocumentFilter.values(), FileExtExecutableFilter.values()) + .flatMap(arr -> Stream.of(arr)) + .flatMap(filter -> filter.getFilter().stream().map(ext -> Pair.of(ext, filter))) + .collect(Collectors.groupingBy( + pair -> pair.getKey(), + Collectors.mapping(pair -> pair.getValue(), + Collectors.toSet()))); private static final String FILE_VIEW_EXT_TYPE_ID = "FILE_VIEW_BY_EXT"; @@ -132,8 +140,7 @@ public class ViewsDAO extends AbstractDAO { } FileTypeExtensionsEvent extEvt = (FileTypeExtensionsEvent) eventData; - String extension = extEvt.getExtension().toLowerCase(); - return key.getFilter().getFilter().contains(extension) + return key.getFilter().equals(extEvt.getExtensionFilter()) && (key.getDataSourceId() == null || key.getDataSourceId().equals(extEvt.getDataSourceId())); } @@ -336,12 +343,7 @@ public class ViewsDAO extends AbstractDAO { ? TreeDisplayCount.INDETERMINATE : TreeDisplayCount.getDeterminate(entry.getValue()); - return new TreeItemDTO<>( - FileTypeExtensionsSearchParams.getTypeId(), - new FileTypeExtensionsSearchParams(entry.getKey(), dataSourceId), - entry.getKey(), - entry.getKey().getDisplayName(), - displayCount); + return createExtensionTreeItem(entry.getKey(), dataSourceId, displayCount); }) .sorted((a, b) -> a.getDisplayName().compareToIgnoreCase(b.getDisplayName())) .collect(Collectors.toList()); @@ -349,6 +351,24 @@ public class ViewsDAO extends AbstractDAO { return new TreeResultsDTO<>(treeList); } + /** + * Creates an extension tree item. + * + * @param filter The extension filter. + * @param dataSourceId The data source id or null. + * @param displayCount The count to display. + * + * @return The extension tree item. + */ + private TreeItemDTO createExtensionTreeItem(FileExtSearchFilter filter, Long dataSourceId, TreeDisplayCount displayCount) { + return new TreeItemDTO<>( + FileTypeExtensionsSearchParams.getTypeId(), + new FileTypeExtensionsSearchParams(filter, dataSourceId), + filter, + filter.getDisplayName(), + displayCount); + } + /** * Returns counts for file size categories. * @@ -384,12 +404,7 @@ public class ViewsDAO extends AbstractDAO { ? TreeDisplayCount.INDETERMINATE : TreeDisplayCount.getDeterminate(entry.getValue()); - return new TreeItemDTO<>( - FileTypeSizeSearchParams.getTypeId(), - new FileTypeSizeSearchParams(entry.getKey(), dataSourceId), - entry.getKey(), - entry.getKey().getDisplayName(), - displayCount); + return createSizeTreeItem(entry.getKey(), dataSourceId, displayCount); }) .sorted((a, b) -> a.getDisplayName().compareToIgnoreCase(b.getDisplayName())) .collect(Collectors.toList()); @@ -397,6 +412,24 @@ public class ViewsDAO extends AbstractDAO { return new TreeResultsDTO<>(treeList); } + /** + * Creates a size tree item. + * + * @param filter The file size filter. + * @param dataSourceId The data source id. + * @param displayCount The display count. + * + * @return The tree item. + */ + private TreeItemDTO createSizeTreeItem(FileSizeFilter filter, Long dataSourceId, TreeDisplayCount displayCount) { + return new TreeItemDTO<>( + FileTypeSizeSearchParams.getTypeId(), + new FileTypeSizeSearchParams(filter, dataSourceId), + filter, + filter.getDisplayName(), + displayCount); + } + /** * Returns counts for file mime type categories. * @@ -489,12 +522,7 @@ public class ViewsDAO extends AbstractDAO { ? TreeDisplayCount.INDETERMINATE : TreeDisplayCount.getDeterminate(entry.getValue()); - return new TreeItemDTO<>( - FileTypeMimeSearchParams.getTypeId(), - new FileTypeMimeSearchParams(entry.getKey(), dataSourceId), - name, - name, - displayCount); + return createMimeTreeItem(entry.getKey(), name, dataSourceId, displayCount); }) .sorted((a, b) -> stringCompare(a.getSearchParams().getMimeType(), b.getSearchParams().getMimeType())) .collect(Collectors.toList()); @@ -508,6 +536,26 @@ public class ViewsDAO extends AbstractDAO { } } + /** + * Creates a mime type tree item. + * + * @param fullMime The full mime type. + * @param mimeName The mime type segment that will be displayed (suffix + * or prefix). + * @param dataSourceId The data source id. + * @param displayCount The count to display. + * + * @return The created tree item. + */ + private TreeItemDTO createMimeTreeItem(String fullMime, String mimeName, Long dataSourceId, TreeDisplayCount displayCount) { + return new TreeItemDTO<>( + FileTypeMimeSearchParams.getTypeId(), + new FileTypeMimeSearchParams(fullMime, dataSourceId), + mimeName, + mimeName, + displayCount); + } + /** * Provides case insensitive comparator integer for strings that may be * null. @@ -667,11 +715,6 @@ public class ViewsDAO extends AbstractDAO { return new BaseSearchResultsDTO(FILE_VIEW_EXT_TYPE_ID, displayName, FileSystemColumnUtils.getColumnKeysForAbstractfile(), fileRows, AbstractFile.class.getName(), startItem, totalResultsCount); } - @Override - void clearCaches() { - this.searchParamsCache.invalidateAll(); - } - private Pair getMimePieces(String mimeType) { int idx = mimeType.indexOf("/"); String mimePrefix = idx > 0 ? mimeType.substring(0, idx) : mimeType; @@ -680,15 +723,36 @@ public class ViewsDAO extends AbstractDAO { } @Override - Set handleIngestComplete() { - // GVDTODO - return Collections.emptySet(); + void clearCaches() { + this.searchParamsCache.invalidateAll(); + handleIngestComplete(); + } + + private TreeItemDTO createTreeItem(DAOEvent daoEvent, TreeDisplayCount count) { + if (daoEvent instanceof FileTypeExtensionsEvent) { + FileTypeExtensionsEvent extEvt = (FileTypeExtensionsEvent) daoEvent; + return createExtensionTreeItem(extEvt.getExtensionFilter(), extEvt.getDataSourceId(), count); + } else if (daoEvent instanceof FileTypeMimeEvent) { + FileTypeMimeEvent mimeEvt = (FileTypeMimeEvent) daoEvent; + return createMimeTreeItem(mimeEvt.getMimeType(), mimeEvt.getDataSourceId(), count); + } else if (daoEvent instanceof FileTypeSizeEvent) { + FileTypeSizeEvent sizeEvt = (FileTypeSizeEvent) daoEvent; + return createSizeTreeItem(sizeEvt.getSizeFilter(), sizeEvt.getDataSourceId(), count); + } else { + return null; + } + } + + @Override + Set handleIngestComplete() { + return getIngestCompleteEvents(this.treeCounts, + (daoEvt, count) -> createTreeItem(daoEvt, count)); } @Override Set shouldRefreshTree() { - // GVDTODO - return Collections.emptySet(); + return getRefreshEvents(this.treeCounts, + (daoEvt, count) -> createTreeItem(daoEvt, count)); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java index dadac922b1..21a389975a 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.mainui.datamodel.events; import java.util.Objects; +import org.sleuthkit.autopsy.mainui.datamodel.FileExtSearchFilter; /** * An event to signal that files have been added or removed @@ -26,16 +27,16 @@ import java.util.Objects; */ public class FileTypeExtensionsEvent implements DAOEvent { - private final String extension; + private final FileExtSearchFilter extensionFilter; private final long dataSourceId; - public FileTypeExtensionsEvent(String extension, long dataSourceId) { - this.extension = extension; + public FileTypeExtensionsEvent(FileExtSearchFilter extensionFilter, long dataSourceId) { + this.extensionFilter = extensionFilter; this.dataSourceId = dataSourceId; } - public String getExtension() { - return extension; + public FileExtSearchFilter getExtensionFilter() { + return extensionFilter; } public long getDataSourceId() { @@ -45,8 +46,8 @@ public class FileTypeExtensionsEvent implements DAOEvent { @Override public int hashCode() { int hash = 7; - hash = 59 * hash + Objects.hashCode(this.extension); - hash = 59 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); + hash = 83 * hash + Objects.hashCode(this.extensionFilter); + hash = 83 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); return hash; } @@ -65,12 +66,14 @@ public class FileTypeExtensionsEvent implements DAOEvent { if (this.dataSourceId != other.dataSourceId) { return false; } - if (!Objects.equals(this.extension, other.extension)) { + if (!Objects.equals(this.extensionFilter, other.extensionFilter)) { return false; } return true; } + + @Override public Type getType() { return Type.RESULT; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java index 96b884a432..7e5fdc5b9e 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java @@ -25,16 +25,22 @@ import java.util.Objects; */ public class FileTypeMimeEvent implements DAOEvent { - private final String mimeType; + private final String mimeTypePrefix; + private final String mimeTypeSuffix; private final long dataSourceId; - public FileTypeMimeEvent(String mimeType, long dataSourceId) { - this.mimeType = mimeType; + public FileTypeMimeEvent(String mimeTypePrefix, String mimeTypeSuffix, long dataSourceId) { + this.mimeTypePrefix = mimeTypePrefix; + this.mimeTypeSuffix = mimeTypeSuffix; this.dataSourceId = dataSourceId; } - public String getMimeType() { - return mimeType; + public String getMimeTypePrefix() { + return mimeTypePrefix; + } + + public String getMimeTypeSuffix() { + return mimeTypeSuffix; } public long getDataSourceId() { @@ -44,8 +50,9 @@ public class FileTypeMimeEvent implements DAOEvent { @Override public int hashCode() { int hash = 7; - hash = 29 * hash + Objects.hashCode(this.mimeType); - hash = 29 * hash + Objects.hashCode(this.dataSourceId); + hash = 31 * hash + Objects.hashCode(this.mimeTypePrefix); + hash = 31 * hash + Objects.hashCode(this.mimeTypeSuffix); + hash = 31 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); return hash; } @@ -61,15 +68,20 @@ public class FileTypeMimeEvent implements DAOEvent { return false; } final FileTypeMimeEvent other = (FileTypeMimeEvent) obj; - if (!Objects.equals(this.mimeType, other.mimeType)) { + if (this.dataSourceId != other.dataSourceId) { return false; } - if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { + if (!Objects.equals(this.mimeTypePrefix, other.mimeTypePrefix)) { + return false; + } + if (!Objects.equals(this.mimeTypeSuffix, other.mimeTypeSuffix)) { return false; } return true; } + + @Override public Type getType() { return Type.RESULT; From ea0fd1c32e1e18057886708bfe02f7702672a520 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 2 Dec 2021 12:41:20 -0500 Subject: [PATCH 09/24] views dao fixes --- .../autopsy/mainui/datamodel/AbstractDAO.java | 55 +++++---- .../autopsy/mainui/datamodel/ViewsDAO.java | 115 +++++++++--------- .../datamodel/events/FileTypeMimeEvent.java | 30 ++--- 3 files changed, 98 insertions(+), 102 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java index 93f8fb7f03..d7964d1d40 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java @@ -21,13 +21,12 @@ package org.sleuthkit.autopsy.mainui.datamodel; import com.google.common.cache.Cache; import org.sleuthkit.autopsy.mainui.datamodel.events.DAOEvent; import java.beans.PropertyChangeEvent; -import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeDisplayCount; @@ -87,40 +86,40 @@ abstract class AbstractDAO { * no data source filtering). * @param itemDataSourceMapping The event digest. */ - static void invalidateKeys(Cache, ?> cache, Function> getKeys, Map> itemDataSourceMapping) { - invalidateKeys(cache, getKeys, Collections.singletonList(itemDataSourceMapping)); + static void invalidateKeys(Cache, ?> cache, Function> getKeys, Map> itemDsMapping) { + invalidateKeys(cache, (keyParams) -> { + Pair pairItems = getKeys.apply(keyParams); + T searchParamsKey = pairItems.getLeft(); + Long searchParamsDsId = pairItems.getRight(); + Set dsIds = itemDsMapping.get(searchParamsKey); + return (dsIds != null && (searchParamsDsId == null || dsIds.contains(searchParamsDsId))); + }); } /** - * Using a digest of event information, clears keys in a cache that may be - * effected by events. + * Determines what keys should be kept in the cache while iterating through + * all the keys. * - * @param cache The cache. - * @param getKeys Using a key from a cache, provides a tuple - * of the relevant key in the data source - * mapping and the data source id (or null if - * no data source filtering). - * @param itemDataSourceMapping The list of event digests. + * @param cache The cache. + * @param shouldInvalidate If the key should be removed from the cache. */ - static void invalidateKeys(Cache, ?> cache, Function> getKeys, List>> itemDataSourceMapping) { + static void invalidateKeys(Cache, ?> cache, Predicate shouldInvalidate) { ConcurrentMap, ?> concurrentMap = cache.asMap(); concurrentMap.forEach((k, v) -> { - Pair pairItems = getKeys.apply(k.getParamData()); - T searchParamsKey = pairItems.getLeft(); - Long searchParamsDsId = pairItems.getRight(); - for (Map> itemDsMapping : itemDataSourceMapping) { - Set dsIds = itemDsMapping.get(searchParamsKey); - if (dsIds != null && (searchParamsDsId == null || dsIds.contains(searchParamsDsId))) { - concurrentMap.remove(k); - } + if (shouldInvalidate.test(k.getParamData())) { + concurrentMap.remove(k); } }); } /** - * Returns a set of tree events gathered from the TreeCounts instance after calling flushEvents. + * Returns a set of tree events gathered from the TreeCounts instance after + * calling flushEvents. + * * @param treeCounts The tree counts instance. - * @param converter The means of acquiring a tree item dto to be placed in the TreeEvent. + * @param converter The means of acquiring a tree item dto to be placed in + * the TreeEvent. + * * @return The generated tree events. */ static Set getIngestCompleteEvents(TreeCounts treeCounts, BiFunction> converter) { @@ -128,11 +127,15 @@ abstract class AbstractDAO { .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) .collect(Collectors.toSet()); } - + /** - * Returns a set of tree events gathered from the TreeCounts instance after calling getEventTimeouts. + * Returns a set of tree events gathered from the TreeCounts instance after + * calling getEventTimeouts. + * * @param treeCounts The tree counts instance. - * @param converter The means of acquiring a tree item dto to be placed in the TreeEvent. + * @param converter The means of acquiring a tree item dto to be placed in + * the TreeEvent. + * * @return The generated tree events. */ static Set getRefreshEvents(TreeCounts treeCounts, BiFunction> converter) { diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index 90e8a9304d..d578f094cd 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -49,6 +50,8 @@ import static org.sleuthkit.autopsy.core.UserPreferences.hideKnownFilesInViewsTr import static org.sleuthkit.autopsy.core.UserPreferences.hideSlackFilesInViewsTree; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeDisplayCount; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; +import org.sleuthkit.autopsy.mainui.datamodel.events.AnalysisResultEvent; +import org.sleuthkit.autopsy.mainui.datamodel.events.AnalysisResultSetEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.DAOEventUtils; import org.sleuthkit.autopsy.mainui.datamodel.events.FileTypeExtensionsEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileTypeMimeEvent; @@ -734,7 +737,9 @@ public class ViewsDAO extends AbstractDAO { return createExtensionTreeItem(extEvt.getExtensionFilter(), extEvt.getDataSourceId(), count); } else if (daoEvent instanceof FileTypeMimeEvent) { FileTypeMimeEvent mimeEvt = (FileTypeMimeEvent) daoEvent; - return createMimeTreeItem(mimeEvt.getMimeType(), mimeEvt.getDataSourceId(), count); + Pair mimePieces = getMimePieces(mimeEvt.getMimeType()); + String mimeName = mimePieces.getRight() == null ? mimePieces.getLeft() : mimePieces.getRight(); + return createMimeTreeItem(mimeEvt.getMimeType(), mimeName, mimeEvt.getDataSourceId(), count); } else if (daoEvent instanceof FileTypeSizeEvent) { FileTypeSizeEvent sizeEvt = (FileTypeSizeEvent) daoEvent; return createSizeTreeItem(sizeEvt.getSizeFilter(), sizeEvt.getDataSourceId(), count); @@ -757,51 +762,56 @@ public class ViewsDAO extends AbstractDAO { @Override Set processEvent(PropertyChangeEvent evt) { - // GVDTODO maps may not be necessary now that this isn't processing a list of events. - Map> fileExtensionDsMap = new HashMap<>(); - Map>> mimeTypeDsMap = new HashMap<>(); - Map> fileSizeDsMap = new HashMap<>(); - AbstractFile af = DAOEventUtils.getFileFromFileEvent(evt); if (af == null) { return Collections.emptySet(); } + long dsId = af.getDataSourceObjectId(); + // create an extension mapping if extension present - if (!StringUtils.isBlank(af.getNameExtension())) { - fileExtensionDsMap - .computeIfAbsent("." + af.getNameExtension(), (k) -> new HashSet<>()) - .add(af.getDataSourceObjectId()); - } + Set fileExtensionDsSet = StringUtils.isBlank(af.getNameExtension()) + ? Collections.emptySet() + : EXTENSION_FILTER_MAP.getOrDefault("." + af.getNameExtension(), Collections.emptySet()); // create a mime type mapping if mime type present - if (!StringUtils.isBlank(af.getMIMEType())) { - Pair mimePieces = getMimePieces(af.getMIMEType()); - mimeTypeDsMap - .computeIfAbsent(mimePieces.getKey(), (k) -> new HashMap<>()) - .computeIfAbsent(mimePieces.getValue(), (k) -> new HashSet<>()) - .add(af.getDataSourceObjectId()); - } + String evtMimeType = StringUtils.isBlank(af.getMIMEType()) ? null : af.getMIMEType(); - // create a size mapping if size present - FileSizeFilter sizeFilter = Stream.of(FileSizeFilter.values()) + // create a size mapping if size present in filters + FileSizeFilter evtFileSize = Stream.of(FileSizeFilter.values()) .filter(filter -> af.getSize() >= filter.getMinBound() && (filter.getMaxBound() == null || af.getSize() < filter.getMaxBound())) .findFirst() .orElse(null); - if (sizeFilter != null) { - fileSizeDsMap - .computeIfAbsent(sizeFilter, (k) -> new HashSet<>()) - .add(af.getDataSourceObjectId()); - } - - if (fileExtensionDsMap.isEmpty() && mimeTypeDsMap.isEmpty() && fileSizeDsMap.isEmpty()) { + if (fileExtensionDsSet.isEmpty() && evtMimeType == null && evtFileSize == null) { return Collections.emptySet(); } - clearRelevantCacheEntries(fileExtensionDsMap, mimeTypeDsMap, fileSizeDsMap); + invalidateKeys(this.searchParamsCache, (searchParams) -> { + if (searchParams instanceof FileTypeExtensionsSearchParams) { + FileTypeExtensionsSearchParams extParams = (FileTypeExtensionsSearchParams) searchParams; + return fileExtensionDsSet.contains(extParams.getFilter()) + && (extParams.getDataSourceId() == null || Objects.equals(extParams.getDataSourceId(), dsId)); - return getDAOEvents(fileExtensionDsMap, mimeTypeDsMap, fileSizeDsMap); + } else if (searchParams instanceof FileTypeMimeSearchParams) { + FileTypeMimeSearchParams mimeParams = (FileTypeMimeSearchParams) searchParams; + return evtMimeType != null && evtMimeType.startsWith(mimeParams.getMimeType()) + && (mimeParams.getDataSourceId() == null || Objects.equals(mimeParams.getDataSourceId(), dsId)); + + } else if (searchParams instanceof FileTypeSizeSearchParams) { + FileTypeSizeSearchParams sizeParams = (FileTypeSizeSearchParams) searchParams; + return Objects.equals(sizeParams.getSizeFilter(), evtFileSize) + && (sizeParams.getDataSourceId() == null || Objects.equals(sizeParams.getDataSourceId(), dsId)); + } else { + return false; + } + }); + + return getDAOEvents(fileExtensionDsSet, evtMimeType, evtFileSize, dsId); + } + + private boolean isInDsFilter(Long dsFilter, Long ds) { + return dsFilter == null || Objects.equals(dsFilter, ds); } /** @@ -809,38 +819,32 @@ public class ViewsDAO extends AbstractDAO { * Clears relevant cache entries from cache based on digest of autopsy * events. * - * @param fileExtensionDsMap Maps the file extension to the data sources - * where files were found with that extension. - * @param mimeTypeDsMap Maps the mime type to the data sources where - * files were found with that mime type. - * @param fileSizeDsMap Maps the size to the data sources where files * * @return The list of affected dao events. */ - private Set getDAOEvents(Map> fileExtensionDsMap, - Map>> mimeTypeDsMap, - Map> fileSizeDsMap) { + private Set getDAOEvents(Set extFilters, + String mimeType, + FileSizeFilter sizeFilter, + long dsId) { - Stream fileExtStream = fileExtensionDsMap.entrySet().stream() - .flatMap(entry -> entry.getValue().stream().map(dsId -> new FileTypeExtensionsEvent(entry.getKey(), dsId))); - - Set fileMimeList = new HashSet<>(); - for (Entry>> prefixEntry : mimeTypeDsMap.entrySet()) { - String mimePrefix = prefixEntry.getKey(); - for (Entry> suffixEntry : prefixEntry.getValue().entrySet()) { - String mimeSuffix = suffixEntry.getKey(); - for (long dsId : suffixEntry.getValue()) { - String mimeType = mimePrefix + (mimeSuffix == null ? "" : ("/" + mimeSuffix)); - fileMimeList.add(new FileTypeMimeEvent(mimeType, dsId)); - } - } + List daoEvents = extFilters.stream() + .map(extFilter -> new FileTypeExtensionsEvent(extFilter, dsId)) + .collect(Collectors.toList()); + + if (mimeType != null) { + daoEvents.add(new FileTypeMimeEvent(mimeType, dsId)); } + + if (sizeFilter != null) { + daoEvents.add(new FileTypeSizeEvent(sizeFilter, dsId)); + } + + List treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() + .map(daoEvt -> new TreeEvent(createTreeItem(daoEvt, TreeDisplayCount.INDETERMINATE), false)) + .collect(Collectors.toList()); - Stream fileSizeStream = fileSizeDsMap.entrySet().stream() - .flatMap(entry -> entry.getValue().stream().map(dsId -> new FileTypeSizeEvent(entry.getKey(), dsId))); - - return Stream.of(fileExtStream, fileMimeList.stream(), fileSizeStream) - .flatMap(stream -> stream) + return Stream.of(daoEvents, treeEvents) + .flatMap(lst -> lst.stream()) .collect(Collectors.toSet()); } @@ -904,6 +908,7 @@ public class ViewsDAO extends AbstractDAO { } } }); + } /** diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java index 7e5fdc5b9e..96b884a432 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeMimeEvent.java @@ -25,22 +25,16 @@ import java.util.Objects; */ public class FileTypeMimeEvent implements DAOEvent { - private final String mimeTypePrefix; - private final String mimeTypeSuffix; + private final String mimeType; private final long dataSourceId; - public FileTypeMimeEvent(String mimeTypePrefix, String mimeTypeSuffix, long dataSourceId) { - this.mimeTypePrefix = mimeTypePrefix; - this.mimeTypeSuffix = mimeTypeSuffix; + public FileTypeMimeEvent(String mimeType, long dataSourceId) { + this.mimeType = mimeType; this.dataSourceId = dataSourceId; } - public String getMimeTypePrefix() { - return mimeTypePrefix; - } - - public String getMimeTypeSuffix() { - return mimeTypeSuffix; + public String getMimeType() { + return mimeType; } public long getDataSourceId() { @@ -50,9 +44,8 @@ public class FileTypeMimeEvent implements DAOEvent { @Override public int hashCode() { int hash = 7; - hash = 31 * hash + Objects.hashCode(this.mimeTypePrefix); - hash = 31 * hash + Objects.hashCode(this.mimeTypeSuffix); - hash = 31 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); + hash = 29 * hash + Objects.hashCode(this.mimeType); + hash = 29 * hash + Objects.hashCode(this.dataSourceId); return hash; } @@ -68,20 +61,15 @@ public class FileTypeMimeEvent implements DAOEvent { return false; } final FileTypeMimeEvent other = (FileTypeMimeEvent) obj; - if (this.dataSourceId != other.dataSourceId) { + if (!Objects.equals(this.mimeType, other.mimeType)) { return false; } - if (!Objects.equals(this.mimeTypePrefix, other.mimeTypePrefix)) { - return false; - } - if (!Objects.equals(this.mimeTypeSuffix, other.mimeTypeSuffix)) { + if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { return false; } return true; } - - @Override public Type getType() { return Type.RESULT; From 70b44834a7b08a3e88973e32b841f29e57bc5813 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 2 Dec 2021 13:01:51 -0500 Subject: [PATCH 10/24] fix compile issues --- .../autopsy/mainui/datamodel/AbstractDAO.java | 2 +- .../autopsy/mainui/datamodel/ViewsDAO.java | 136 +++++------------- 2 files changed, 40 insertions(+), 98 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java index d7964d1d40..3123dccc62 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java @@ -103,7 +103,7 @@ abstract class AbstractDAO { * @param cache The cache. * @param shouldInvalidate If the key should be removed from the cache. */ - static void invalidateKeys(Cache, ?> cache, Predicate shouldInvalidate) { + static void invalidateKeys(Cache, ?> cache, Predicate shouldInvalidate) { ConcurrentMap, ?> concurrentMap = cache.asMap(); concurrentMap.forEach((k, v) -> { if (shouldInvalidate.test(k.getParamData())) { diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index d578f094cd..26134d71f4 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -50,8 +50,6 @@ import static org.sleuthkit.autopsy.core.UserPreferences.hideKnownFilesInViewsTr import static org.sleuthkit.autopsy.core.UserPreferences.hideSlackFilesInViewsTree; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeDisplayCount; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; -import org.sleuthkit.autopsy.mainui.datamodel.events.AnalysisResultEvent; -import org.sleuthkit.autopsy.mainui.datamodel.events.AnalysisResultSetEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.DAOEventUtils; import org.sleuthkit.autopsy.mainui.datamodel.events.FileTypeExtensionsEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileTypeMimeEvent; @@ -97,7 +95,7 @@ public class ViewsDAO extends AbstractDAO { return instance; } - private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); + private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); private final TreeCounts treeCounts = new TreeCounts<>(); private SleuthkitCase getCase() throws NoCurrentCaseException { @@ -111,7 +109,7 @@ public class ViewsDAO extends AbstractDAO { throw new IllegalArgumentException("Data source id must be greater than 0 or null"); } - SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); + SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); return searchParamsCache.get(searchParams, () -> fetchExtensionSearchResultsDTOs(key.getFilter(), key.getDataSourceId(), startItem, maxCount)); } @@ -122,7 +120,7 @@ public class ViewsDAO extends AbstractDAO { throw new IllegalArgumentException("Data source id must be greater than 0 or null"); } - SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); + SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); return searchParamsCache.get(searchParams, () -> fetchMimeSearchResultsDTOs(key.getMimeType(), key.getDataSourceId(), startItem, maxCount)); } @@ -133,7 +131,7 @@ public class ViewsDAO extends AbstractDAO { throw new IllegalArgumentException("Data source id must be greater than 0 or null"); } - SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); + SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); return searchParamsCache.get(searchParams, () -> fetchSizeSearchResultsDTOs(key.getSizeFilter(), key.getDataSourceId(), startItem, maxCount)); } @@ -770,7 +768,7 @@ public class ViewsDAO extends AbstractDAO { long dsId = af.getDataSourceObjectId(); // create an extension mapping if extension present - Set fileExtensionDsSet = StringUtils.isBlank(af.getNameExtension()) + Set evtExtFilters = StringUtils.isBlank(af.getNameExtension()) ? Collections.emptySet() : EXTENSION_FILTER_MAP.getOrDefault("." + af.getNameExtension(), Collections.emptySet()); @@ -783,42 +781,49 @@ public class ViewsDAO extends AbstractDAO { .findFirst() .orElse(null); - if (fileExtensionDsSet.isEmpty() && evtMimeType == null && evtFileSize == null) { + if (evtExtFilters.isEmpty() && evtMimeType == null && evtFileSize == null) { return Collections.emptySet(); } - invalidateKeys(this.searchParamsCache, (searchParams) -> { - if (searchParams instanceof FileTypeExtensionsSearchParams) { - FileTypeExtensionsSearchParams extParams = (FileTypeExtensionsSearchParams) searchParams; - return fileExtensionDsSet.contains(extParams.getFilter()) - && (extParams.getDataSourceId() == null || Objects.equals(extParams.getDataSourceId(), dsId)); + invalidateKeys(this.searchParamsCache, + (Predicate) (searchParams) -> searchParamsMatchEvent(evtExtFilters, evtMimeType, evtFileSize, dsId, searchParams)); - } else if (searchParams instanceof FileTypeMimeSearchParams) { - FileTypeMimeSearchParams mimeParams = (FileTypeMimeSearchParams) searchParams; - return evtMimeType != null && evtMimeType.startsWith(mimeParams.getMimeType()) - && (mimeParams.getDataSourceId() == null || Objects.equals(mimeParams.getDataSourceId(), dsId)); - - } else if (searchParams instanceof FileTypeSizeSearchParams) { - FileTypeSizeSearchParams sizeParams = (FileTypeSizeSearchParams) searchParams; - return Objects.equals(sizeParams.getSizeFilter(), evtFileSize) - && (sizeParams.getDataSourceId() == null || Objects.equals(sizeParams.getDataSourceId(), dsId)); - } else { - return false; - } - }); - - return getDAOEvents(fileExtensionDsSet, evtMimeType, evtFileSize, dsId); + return getDAOEvents(evtExtFilters, evtMimeType, evtFileSize, dsId); } - private boolean isInDsFilter(Long dsFilter, Long ds) { - return dsFilter == null || Objects.equals(dsFilter, ds); + private boolean searchParamsMatchEvent(Set evtExtFilters, + String evtMimeType, + FileSizeFilter evtFileSize, + long dsId, + Object searchParams) { + + if (searchParams instanceof FileTypeExtensionsSearchParams) { + FileTypeExtensionsSearchParams extParams = (FileTypeExtensionsSearchParams) searchParams; + return evtExtFilters.contains(extParams.getFilter()) + && (extParams.getDataSourceId() == null || Objects.equals(extParams.getDataSourceId(), dsId)); + + } else if (searchParams instanceof FileTypeMimeSearchParams) { + FileTypeMimeSearchParams mimeParams = (FileTypeMimeSearchParams) searchParams; + return evtMimeType != null && evtMimeType.startsWith(mimeParams.getMimeType()) + && (mimeParams.getDataSourceId() == null || Objects.equals(mimeParams.getDataSourceId(), dsId)); + + } else if (searchParams instanceof FileTypeSizeSearchParams) { + FileTypeSizeSearchParams sizeParams = (FileTypeSizeSearchParams) searchParams; + return Objects.equals(sizeParams.getSizeFilter(), evtFileSize) + && (sizeParams.getDataSourceId() == null || Objects.equals(sizeParams.getDataSourceId(), dsId)); + } else { + return false; + } } /** - * * Clears relevant cache entries from cache based on digest of autopsy * events. * + * @param extFilters The set of affected extension filters. + * @param mimeType The affected mime type or null. + * @param sizeFilter The affected size filter or null. + * @param dsId The file object id. * * @return The list of affected dao events. */ @@ -830,15 +835,15 @@ public class ViewsDAO extends AbstractDAO { List daoEvents = extFilters.stream() .map(extFilter -> new FileTypeExtensionsEvent(extFilter, dsId)) .collect(Collectors.toList()); - + if (mimeType != null) { daoEvents.add(new FileTypeMimeEvent(mimeType, dsId)); } - + if (sizeFilter != null) { daoEvents.add(new FileTypeSizeEvent(sizeFilter, dsId)); } - + List treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() .map(daoEvt -> new TreeEvent(createTreeItem(daoEvt, TreeDisplayCount.INDETERMINATE), false)) .collect(Collectors.toList()); @@ -848,69 +853,6 @@ public class ViewsDAO extends AbstractDAO { .collect(Collectors.toSet()); } - /** - * Clears relevant cache entries from cache based on digest of autopsy - * events. - * - * @param fileExtensionDsMap Maps the file extension to the data sources - * where files were found with that extension. - * @param mimeTypeDsMap Maps the mime type to the data sources where - * files were found with that mime type. - * @param fileSizeDsMap Maps the size to the data sources where files - * were found within that size filter. - */ - private void clearRelevantCacheEntries(Map> fileExtensionDsMap, - Map>> mimeTypeDsMap, - Map> fileSizeDsMap) { - - // invalidate cache entries that are affected by events - ConcurrentMap, SearchResultsDTO> concurrentMap = this.searchParamsCache.asMap(); - concurrentMap.forEach((k, v) -> { - Object baseParams = k.getParamData(); - if (baseParams instanceof FileTypeExtensionsSearchParams) { - FileTypeExtensionsSearchParams extParams = (FileTypeExtensionsSearchParams) baseParams; - // if search params have a filter where extension is present and the data source id is null or == - boolean isMatch = extParams.getFilter().getFilter().stream().anyMatch((ext) -> { - Set dsIds = fileExtensionDsMap.get(ext); - return (dsIds != null && (extParams.getDataSourceId() == null || dsIds.contains(extParams.getDataSourceId()))); - }); - - if (isMatch) { - concurrentMap.remove(k); - } - } else if (baseParams instanceof FileTypeMimeSearchParams) { - FileTypeMimeSearchParams mimeParams = (FileTypeMimeSearchParams) baseParams; - Pair mimePieces = getMimePieces(mimeParams.getMimeType()); - Map> suffixes = mimeTypeDsMap.get(mimePieces.getKey()); - if (suffixes == null) { - return; - } - - // if search params is top level mime prefix (without suffix) and data source is null or ==. - if (mimePieces.getValue() == null - && (mimeParams.getDataSourceId() == null - || suffixes.values().stream().flatMap(set -> set.stream()).anyMatch(ds -> Objects.equals(mimeParams.getDataSourceId(), ds)))) { - - concurrentMap.remove(k); - // otherwise, see if suffix is present - } else { - Set dataSources = suffixes.get(mimePieces.getValue()); - if (dataSources != null && (mimeParams.getDataSourceId() == null || dataSources.contains(mimeParams.getDataSourceId()))) { - concurrentMap.remove(k); - } - } - - } else if (baseParams instanceof FileTypeSizeSearchParams) { - FileTypeSizeSearchParams sizeParams = (FileTypeSizeSearchParams) baseParams; - Set dataSources = fileSizeDsMap.get(sizeParams.getSizeFilter()); - if (dataSources != null && (sizeParams.getDataSourceId() == null || dataSources.contains(sizeParams.getDataSourceId()))) { - concurrentMap.remove(k); - } - } - }); - - } - /** * Handles fetching and paging of data for file types by extension. */ From 4991549da2c075a1a0eb9ec001acf1e20171485f Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 2 Dec 2021 14:24:46 -0500 Subject: [PATCH 11/24] os account and tags dao updates --- .../mainui/datamodel/OsAccountsDAO.java | 8 +- .../autopsy/mainui/datamodel/TagsDAO.java | 98 ++++++++----------- .../mainui/datamodel/events/TagsEvent.java | 18 ++-- 3 files changed, 57 insertions(+), 67 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java index 78020b4dbf..e86ad03f96 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java @@ -187,13 +187,11 @@ public class OsAccountsDAO extends AbstractDAO { @Override Set handleIngestComplete() { - // GVDTODO return Collections.emptySet(); } @Override Set shouldRefreshTree() { - // GVDTODO return Collections.emptySet(); } @@ -202,9 +200,9 @@ public class OsAccountsDAO extends AbstractDAO { if (!OS_EVENTS.contains(evt.getPropertyName())) { return Collections.emptySet(); } - - this.searchParamsCache.invalidateAll(); - + + this.searchParamsCache.invalidateAll(); + return Collections.singleton(new OsAccountEvent()); } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java index 37c23ef375..6aea7c6907 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -27,20 +27,14 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.Objects; import java.util.Set; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.commons.lang3.tuple.Triple; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -52,7 +46,9 @@ import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; import org.sleuthkit.autopsy.mainui.datamodel.TagsSearchParams.TagType; +import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.TagsEvent; +import org.sleuthkit.autopsy.mainui.datamodel.events.TreeCounts; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; import org.sleuthkit.datamodel.AbstractFile; @@ -90,7 +86,6 @@ public class TagsDAO extends AbstractDAO { private static final int CACHE_SIZE = 5; // rule of thumb: 5 entries times number of cached SearchParams sub-types private static final long CACHE_DURATION = 2; private static final TimeUnit CACHE_DURATION_UNITS = TimeUnit.MINUTES; - private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); private static final String USER_NAME_PROPERTY = "user.name"; //NON-NLS @@ -129,6 +124,11 @@ public class TagsDAO extends AbstractDAO { return new ColumnKey(name, name, Bundle.TagsDAO_fileColumns_noDescription()); } + private final Cache, SearchResultsDTO> searchParamsCache + = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); + + private final TreeCounts treeCounts = new TreeCounts<>(); + public SearchResultsDTO getTags(TagsSearchParams key, long startItem, Long maxCount) throws ExecutionException, IllegalArgumentException { if (key.getTagName() == null) { throw new IllegalArgumentException("Must have non-null tag name"); @@ -301,71 +301,59 @@ public class TagsDAO extends AbstractDAO { } TagsEvent tagEvt = (TagsEvent) daoEvt; - return (tagParams.getTagName().getId() == tagEvt.getTagNameId() + return (Objects.equals(tagParams.getTagName(), tagEvt.getTagName()) && tagParams.getTagType().equals(tagEvt.getTagType()) && (tagParams.getDataSourceId() == null || tagEvt.getDataSourceId() == null || tagParams.getDataSourceId() == tagEvt.getDataSourceId())); } - @Override - void clearCaches() { - this.searchParamsCache.invalidateAll(); + private TreeItemDTO getTreeItem(TagsEvent evt, TreeResultsDTO.TreeDisplayCount count) { + return new TreeItemDTO<>( + TagsSearchParams.getTypeId(), + new TagsSearchParams(evt.getTagName(), evt.getTagType(), evt.getDataSourceId()), + evt.getTagName().getId(), + evt.getTagName().getDisplayName(), + count); } @Override - Set handleIngestComplete() { - // GVDTODO - return Collections.emptySet(); + void clearCaches() { + this.searchParamsCache.invalidateAll(); + handleIngestComplete(); + } + + @Override + Set handleIngestComplete() { + return getIngestCompleteEvents(this.treeCounts, (evt, count) -> getTreeItem(evt, count)); } @Override Set shouldRefreshTree() { - // GVDTODO - return Collections.emptySet(); + return getRefreshEvents(this.treeCounts, (evt, count) -> getTreeItem(evt, count)); } @Override Set processEvent(PropertyChangeEvent evt) { - // GVDTODO this may be rewritten simpler now that it isn't processing a list of events - Map, Set>> mapping = new HashMap<>(); - - // tag type, tag name id, data source id (or null if unknown) - Triple data = getTagData(evt); - if (data != null) { - mapping.computeIfAbsent(Pair.of(data.getLeft(), data.getMiddle()), k -> new HashSet<>()) - .add(Optional.ofNullable(data.getRight())); - } - - - // don't continue if no mapping entries - if (mapping.isEmpty()) { + TagsEvent data = getTagData(evt); + if (data == null) { return Collections.emptySet(); } - ConcurrentMap, SearchResultsDTO> concurrentMap = this.searchParamsCache.asMap(); - concurrentMap.forEach((k, v) -> { - TagsSearchParams paramData = k.getParamData(); - Set> affectedDataSources = mapping.get(Pair.of(paramData.getTagType(), paramData.getTagName().getId())); - // we only clear key if the tag name / type line up and either the parameters data source wasn't specified, - // there is a wild card data source for the event, or the data source is contained in the list of data sources - // affected by the event - if (affectedDataSources != null - && (paramData.getDataSourceId() == null - || affectedDataSources.contains(Optional.empty()) - || affectedDataSources.contains(Optional.of(paramData.getDataSourceId())))) { - concurrentMap.remove(k); - } + invalidateKeys(this.searchParamsCache, (searchParams) -> { + return (Objects.equals(searchParams.getTagType(), data.getTagType()) + && Objects.equals(searchParams.getTagName(), data.getTagName()) + && (searchParams.getDataSourceId() == null || Objects.equals(searchParams.getDataSourceId(), data.getDataSourceId()))); }); - return mapping.entrySet().stream() - .flatMap(entry -> { - TagType tagType = entry.getKey().getLeft(); - Long tagNameId = entry.getKey().getRight(); + Collection daoEvents = Collections.singletonList(data); - return entry.getValue().stream() - .map((dsIdOpt) -> new TagsEvent(tagType, tagNameId, dsIdOpt.orElse(null))); - }) + Collection treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() + .map(arEvt -> new TreeEvent(getTreeItem(arEvt, TreeResultsDTO.TreeDisplayCount.INDETERMINATE), false)) + .collect(Collectors.toList()); + + return Stream.of(daoEvents, treeEvents) + .flatMap(lst -> lst.stream()) .collect(Collectors.toSet()); } @@ -378,21 +366,21 @@ public class TagsDAO extends AbstractDAO { * @return tag type, tag name id, data source id (or null if none determined * from event). */ - private Triple getTagData(PropertyChangeEvent evt) { + private TagsEvent getTagData(PropertyChangeEvent evt) { if (evt instanceof BlackBoardArtifactTagAddedEvent) { BlackBoardArtifactTagAddedEvent event = (BlackBoardArtifactTagAddedEvent) evt; // ensure tag added event has a valid content id if (event.getAddedTag() != null && event.getAddedTag().getContent() != null && event.getAddedTag().getArtifact() != null) { - return Triple.of(TagType.RESULT, event.getAddedTag().getName().getId(), event.getAddedTag().getArtifact().getDataSourceObjectID()); + return new TagsEvent(TagType.RESULT, event.getAddedTag().getName(), event.getAddedTag().getArtifact().getDataSourceObjectID()); } } else if (evt instanceof BlackBoardArtifactTagDeletedEvent) { BlackBoardArtifactTagDeletedEvent event = (BlackBoardArtifactTagDeletedEvent) evt; BlackBoardArtifactTagDeletedEvent.DeletedBlackboardArtifactTagInfo deletedTagInfo = event.getDeletedTagInfo(); if (deletedTagInfo != null) { - return Triple.of(TagType.RESULT, deletedTagInfo.getName().getId(), null); + return new TagsEvent(TagType.RESULT, deletedTagInfo.getName(), null); } } else if (evt instanceof ContentTagAddedEvent) { ContentTagAddedEvent event = (ContentTagAddedEvent) evt; @@ -400,14 +388,14 @@ public class TagsDAO extends AbstractDAO { if (event.getAddedTag() != null && event.getAddedTag().getContent() != null) { Content content = event.getAddedTag().getContent(); Long dsId = content instanceof AbstractFile ? ((AbstractFile) content).getDataSourceObjectId() : null; - return Triple.of(TagType.FILE, event.getAddedTag().getName().getId(), dsId); + return new TagsEvent(TagType.FILE, event.getAddedTag().getName(), dsId); } } else if (evt instanceof ContentTagDeletedEvent) { ContentTagDeletedEvent event = (ContentTagDeletedEvent) evt; // ensure tag deleted event has a valid content id ContentTagDeletedEvent.DeletedContentTagInfo deletedTagInfo = event.getDeletedTagInfo(); if (deletedTagInfo != null) { - return Triple.of(TagType.FILE, deletedTagInfo.getName().getId(), null); + return new TagsEvent(TagType.FILE, deletedTagInfo.getName(), null); } } return null; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/TagsEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/TagsEvent.java index 948f96e20d..802303b744 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/TagsEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/TagsEvent.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.mainui.datamodel.events; import java.util.Objects; import org.sleuthkit.autopsy.mainui.datamodel.TagsSearchParams.TagType; +import org.sleuthkit.datamodel.TagName; /** * An event to signal that tags have been added or removed on the @@ -28,21 +29,22 @@ import org.sleuthkit.autopsy.mainui.datamodel.TagsSearchParams.TagType; public class TagsEvent implements DAOEvent { private final TagType type; - private final Long tagNameId; + private final TagName tagName; private final Long dataSourceId; - public TagsEvent(TagType type, Long tagNameId, Long dataSourceId) { + public TagsEvent(TagType type, TagName tagName, Long dataSourceId) { this.type = type; - this.tagNameId = tagNameId; + this.tagName = tagName; this.dataSourceId = dataSourceId; } + public TagType getTagType() { return type; } - public Long getTagNameId() { - return tagNameId; + public TagName getTagName() { + return tagName; } /** @@ -57,7 +59,7 @@ public class TagsEvent implements DAOEvent { public int hashCode() { int hash = 7; hash = 97 * hash + Objects.hashCode(this.type); - hash = 97 * hash + Objects.hashCode(this.tagNameId); + hash = 97 * hash + Objects.hashCode(this.tagName); hash = 97 * hash + Objects.hashCode(this.dataSourceId); return hash; } @@ -77,7 +79,7 @@ public class TagsEvent implements DAOEvent { if (this.type != other.type) { return false; } - if (!Objects.equals(this.tagNameId, other.tagNameId)) { + if (!Objects.equals(this.tagName, other.tagName)) { return false; } if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { @@ -86,6 +88,8 @@ public class TagsEvent implements DAOEvent { return true; } + + @Override public Type getType() { return Type.RESULT; From 74cbbc97e15167e8fcd9713f6f4d42a54a69a56c Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 2 Dec 2021 20:37:01 -0500 Subject: [PATCH 12/24] file system work --- .../mainui/datamodel/AnalysisResultDAO.java | 1 - .../mainui/datamodel/CommAccountsDAO.java | 2 +- .../mainui/datamodel/DataArtifactDAO.java | 2 +- .../mainui/datamodel/FileSystemDAO.java | 271 ++++++++++-------- .../autopsy/mainui/datamodel/ViewsDAO.java | 12 +- .../events/FileSystemContentEvent.java | 23 +- .../nodes/AnalysisResultTypeFactory.java | 17 +- .../mainui/nodes/DataArtifactTypeFactory.java | 8 +- .../mainui/nodes/FileSystemFactory.java | 18 +- .../mainui/nodes/TreeChildFactory.java | 5 +- .../mainui/nodes/ViewsTypeFactory.java | 17 +- 11 files changed, 206 insertions(+), 170 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java index fd98ac1ed4..329241821d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.stream.Collectors; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java index d9de9f4cb7..b36d96683c 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/CommAccountsDAO.java @@ -286,7 +286,7 @@ public class CommAccountsDAO extends AbstractDAO { return Collections.emptySet(); } - super.invalidateKeys(this.searchParamsCache, + invalidateKeys(this.searchParamsCache, (sp) -> Pair.of(sp.getType(), sp.getDataSourceId()), accountTypeMap); List accountEvents = new ArrayList<>(); diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java index e3a3ab8486..4368179a2e 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java @@ -210,7 +210,7 @@ public class DataArtifactDAO extends BlackboardArtifactDAO { return Collections.emptySet(); } - super.invalidateKeys(this.dataArtifactCache, (sp) -> Pair.of(sp.getArtifactType(), sp.getDataSourceId()), artifactTypeDataSourceMap); + invalidateKeys(this.dataArtifactCache, (sp) -> Pair.of(sp.getArtifactType(), sp.getDataSourceId()), artifactTypeDataSourceMap); // gather dao events based on artifacts List dataArtifactEvents = new ArrayList<>(); diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java index c3c9840bae..6c016b16ae 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java @@ -23,11 +23,10 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableSet; import java.beans.PropertyChangeEvent; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentMap; @@ -61,6 +60,7 @@ import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeDisplayCount; import org.sleuthkit.autopsy.mainui.datamodel.events.FileSystemContentEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileSystemHostEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileSystemPersonEvent; +import org.sleuthkit.autopsy.mainui.datamodel.events.TreeCounts; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; import org.sleuthkit.datamodel.AbstractFile; @@ -110,8 +110,11 @@ public class FileSystemDAO extends AbstractDAO { Case.Events.HOSTS_REMOVED_FROM_PERSON.toString() ); - private final Cache, BaseSearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); - + private final Cache, BaseSearchResultsDTO> searchParamsCache + = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); + + private final TreeCounts treeCounts = new TreeCounts<>(); + private static final String FILE_SYSTEM_TYPE_ID = "FILE_SYSTEM"; private static FileSystemDAO instance = null; @@ -130,7 +133,7 @@ public class FileSystemDAO extends AbstractDAO { FileSystemContentEvent contentEvt = (FileSystemContentEvent) daoEvent; - return contentEvt.getContentObjectId() == null || key.getContentObjectId().equals(contentEvt.getContentObjectId()); + return contentEvt.getContentObjectId() == null || Objects.equals(key.getContentObjectId(), contentEvt.getContentObjectId()); } private boolean isSystemHostInvalidating(FileSystemHostSearchParam key, DAOEvent daoEvent) { @@ -315,19 +318,13 @@ public class FileSystemDAO extends AbstractDAO { return searchParamsCache.get(searchParams, () -> fetchHostsForTable(searchParams)); } - @Override - void clearCaches() { - this.searchParamsCache.invalidateAll(); - } - - private Long getHostFromDs(Content dataSource) { + private Host getHostFromDs(Content dataSource) { if (!(dataSource instanceof DataSource)) { return null; } try { - Host host = ((DataSource) dataSource).getHost(); - return host == null ? null : host.getHostId(); + return ((DataSource) dataSource).getHost(); } catch (TskCoreException ex) { logger.log(Level.WARNING, "There was an error getting the host for data source with id: " + dataSource.getId(), ex); return null; @@ -358,24 +355,15 @@ public class FileSystemDAO extends AbstractDAO { return false; } - @Override - Set handleIngestComplete() { - // GVDTODO - return Collections.emptySet(); - } - - @Override - Set shouldRefreshTree() { - // GVDTODO - return Collections.emptySet(); - } - @Override Set processEvent(PropertyChangeEvent evt) { - // GVDTODO these can probably be rewritten now that it isn't handling a collection of autopsy events - Set affectedPersons = new HashSet<>(); - Set affectedHosts = new HashSet<>(); - Set affectedParentContent = new HashSet<>(); + Content affectedParentContent = null; + Host affectedParentHost = null; + + // GVDTODO person parents and parent of persons not handled yet + // optional present but null indicates no person parent + Optional affectedParentPerson = Optional.empty(); + boolean refreshAllContent = false; Content content = DAOEventUtils.getDerivedFileContentFromFileEvent(evt); @@ -388,132 +376,146 @@ public class FileSystemDAO extends AbstractDAO { return Collections.emptySet(); } - if (parentContent == null) { - return Collections.emptySet(); - } - if (invalidatesAllFileSystem(parentContent)) { refreshAllContent = true; } else { - affectedParentContent.add(parentContent.getId()); + affectedParentContent = parentContent; } } else if (evt instanceof DataSourceAddedEvent) { - Long hostId = getHostFromDs(((DataSourceAddedEvent) evt).getDataSource()); - if (hostId != null) { - affectedHosts.add(hostId); - } + Host host = getHostFromDs(((DataSourceAddedEvent) evt).getDataSource()); + affectedParentHost = host; + } else if (evt instanceof DataSourceNameChangedEvent) { - Long hostId = getHostFromDs(((DataSourceNameChangedEvent) evt).getDataSource()); - if (hostId != null) { - affectedHosts.add(hostId); - } + Host host = getHostFromDs(((DataSourceNameChangedEvent) evt).getDataSource()); + affectedParentHost = host; + } else if (evt instanceof HostsAddedEvent) { // GVDTODO how best to handle host added? } else if (evt instanceof HostsUpdatedEvent) { // GVDTODO how best to handle host updated? } else if (evt instanceof HostsAddedToPersonEvent) { Person person = ((HostsAddedToPersonEvent) evt).getPerson(); - affectedPersons.add(person == null ? null : person.getPersonId()); + affectedParentPerson = Optional.of(person); } else if (evt instanceof HostsRemovedFromPersonEvent) { Person person = ((HostsRemovedFromPersonEvent) evt).getPerson(); - affectedPersons.add(person == null ? null : person.getPersonId()); + affectedParentPerson = Optional.of(person); } - final boolean triggerFullRefresh = refreshAllContent; + // if nothing affected, return no events + if (!refreshAllContent && affectedParentContent == null && affectedParentHost == null && !affectedParentPerson.isPresent()) { + return Collections.emptySet(); + } - // GVDTODO handling null ids versus the 'No Persons' option - ConcurrentMap, BaseSearchResultsDTO> concurrentMap = this.searchParamsCache.asMap(); - concurrentMap.forEach((k, v) -> { - Object searchParams = k.getParamData(); - if (searchParams instanceof FileSystemPersonSearchParam) { - FileSystemPersonSearchParam personParam = (FileSystemPersonSearchParam) searchParams; - if (affectedPersons.contains(personParam.getPersonObjectId())) { - concurrentMap.remove(k); - } - } else if (searchParams instanceof FileSystemHostSearchParam) { - FileSystemHostSearchParam hostParams = (FileSystemHostSearchParam) searchParams; - if (affectedHosts.contains(hostParams.getHostObjectId())) { - concurrentMap.remove(k); - } - } else if (searchParams instanceof FileSystemContentSearchParam) { - FileSystemContentSearchParam contentParams = (FileSystemContentSearchParam) searchParams; - if (triggerFullRefresh - || contentParams.getContentObjectId() == null - || affectedParentContent.contains(contentParams.getContentObjectId())) { - concurrentMap.remove(k); - } - } + invalidateKeys(affectedParentPerson, affectedParentHost, affectedParentContent, refreshAllContent); + + return getDAOEvents(affectedParentPerson, affectedParentHost, affectedParentContent, refreshAllContent); + } + + private Set getDAOEvents(Optional affectedPerson, Host affectedHost, Content affectedContent, boolean triggerFullRefresh) { + List daoEvents = new ArrayList<>(); + + if (triggerFullRefresh) { + daoEvents.add(new FileSystemContentEvent(null)); + } else if (affectedContent != null) { + daoEvents.add(new FileSystemContentEvent(affectedContent)); + } + + if (affectedHost != null) { + daoEvents.add(new FileSystemHostEvent(affectedHost.getHostId())); + } + + affectedPerson.ifPresent((person) -> { + daoEvents.add(new FileSystemPersonEvent(person == null ? null : person.getPersonId())); }); - Stream fileEvts = triggerFullRefresh - ? Stream.of(new FileSystemContentEvent(null)) - : affectedParentContent.stream().map(id -> new FileSystemContentEvent(id)); + List treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() + .map(daoEvt -> new TreeEvent(createTreeItem(daoEvt, TreeDisplayCount.INDETERMINATE), false)) + .collect(Collectors.toList()); - return Stream.of( - affectedPersons.stream().map(id -> new FileSystemPersonEvent(id)), - affectedHosts.stream().map(id -> new FileSystemHostEvent(id)), - fileEvts - ) - .flatMap(s -> s) + return Stream.of(daoEvents, treeEvents) + .flatMap(lst -> lst.stream()) .collect(Collectors.toSet()); } - + + private void invalidateKeys(Optional affectedPerson, Host affectedHost, Content affectedContent, boolean triggerFullRefresh) { + ConcurrentMap, ?> concurrentMap = this.searchParamsCache.asMap(); + concurrentMap.forEach((k, v) -> { + Object searchParams = k.getParamData(); + boolean shouldInvalidate = false; + if (searchParams instanceof FileSystemPersonSearchParam && affectedPerson.isPresent()) { + shouldInvalidate = Objects.equals( + ((FileSystemPersonSearchParam) searchParams).getPersonObjectId(), + // to allow for null parent person + affectedPerson.flatMap(p -> Optional.ofNullable(p.getPersonId())).orElse(null) + ); + + } else if (searchParams instanceof FileSystemHostSearchParam && affectedHost != null) { + shouldInvalidate = Objects.equals( + ((FileSystemHostSearchParam) searchParams).getHostObjectId(), + affectedHost.getHostId() + ); + + } else if (searchParams instanceof FileSystemContentSearchParam) { + if (triggerFullRefresh) { + shouldInvalidate = true; + } else if (affectedContent != null) { + shouldInvalidate = Objects.equals( + ((FileSystemContentSearchParam) searchParams).getContentObjectId(), + affectedContent.getId() + ); + } + } + + if (shouldInvalidate) { + concurrentMap.remove(k); + } + }); + } + /** * Get all data sources belonging to a given host. - * + * * @param host The host. - * + * * @return Results containing all data sources for the given host. - * - * @throws ExecutionException + * + * @throws ExecutionException */ public TreeResultsDTO getDataSourcesForHost(Host host) throws ExecutionException { try { List> treeItemRows = new ArrayList<>(); for (DataSource ds : Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getDataSourcesForHost(host)) { - treeItemRows.add(new TreeResultsDTO.TreeItemDTO<>( - FileSystemContentSearchParam.getTypeId(), - new FileSystemContentSearchParam(ds.getId()), - ds, - ds.getName(), - null - )); + treeItemRows.add(createDisplayableContentTreeItem(ds, TreeDisplayCount.NOT_SHOWN)); } return new TreeResultsDTO<>(treeItemRows); } catch (NoCurrentCaseException | TskCoreException ex) { throw new ExecutionException("An error occurred while fetching images for host with ID " + host.getHostId(), ex); } } - + /** * Create results for a single given data source ID (not its children). - * + * * @param dataSourceObjId The data source object ID. - * + * * @return Results containing just this data source. - * - * @throws ExecutionException + * + * @throws ExecutionException */ public TreeResultsDTO getSingleDataSource(long dataSourceObjId) throws ExecutionException { try { List> treeItemRows = new ArrayList<>(); DataSource ds = Case.getCurrentCaseThrows().getSleuthkitCase().getDataSource(dataSourceObjId); - treeItemRows.add(new TreeResultsDTO.TreeItemDTO<>( - FileSystemContentSearchParam.getTypeId(), - new FileSystemContentSearchParam(ds.getId()), - ds, - ds.getName(), - null - )); - + treeItemRows.add(createDisplayableContentTreeItem(ds, TreeDisplayCount.NOT_SHOWN)); return new TreeResultsDTO<>(treeItemRows); } catch (NoCurrentCaseException | TskCoreException | TskDataException ex) { throw new ExecutionException("An error occurred while fetching data source with ID " + dataSourceObjId, ex); } } - + /** - * Get the children that will be displayed in the tree for a given content ID. + * Get the children that will be displayed in the tree for a given content + * ID. * * @param contentId Object ID of parent content. * @@ -523,23 +525,18 @@ public class FileSystemDAO extends AbstractDAO { */ public TreeResultsDTO getDisplayableContentChildren(Long contentId) throws ExecutionException { try { - + List treeChildren = FileSystemColumnUtils.getVisibleTreeNodeChildren(contentId); - + List> treeItemRows = new ArrayList<>(); for (Content child : treeChildren) { Long countForNode = null; if ((child instanceof AbstractFile) - && ! (child instanceof LocalFilesDataSource)) { + && !(child instanceof LocalFilesDataSource)) { countForNode = getContentForTable(new FileSystemContentSearchParam(child.getId()), 0, null).getTotalResultsCount(); } - treeItemRows.add(new TreeResultsDTO.TreeItemDTO<>( - FileSystemContentSearchParam.getTypeId(), - new FileSystemContentSearchParam(child.getId()), - child, - getNameForContent(child), - countForNode == null ? TreeDisplayCount.NOT_SHOWN : TreeDisplayCount.getDeterminate(countForNode) - )); + TreeDisplayCount displayCount = countForNode == null ? TreeDisplayCount.NOT_SHOWN : TreeDisplayCount.getDeterminate(countForNode); + treeItemRows.add(createDisplayableContentTreeItem(child, displayCount)); } return new TreeResultsDTO<>(treeItemRows); @@ -547,23 +544,63 @@ public class FileSystemDAO extends AbstractDAO { throw new ExecutionException("An error occurred while fetching data artifact counts.", ex); } } - + + private TreeResultsDTO.TreeItemDTO createDisplayableContentTreeItem(Content child, TreeDisplayCount displayCount) { + return new TreeResultsDTO.TreeItemDTO<>( + FileSystemContentSearchParam.getTypeId(), + new FileSystemContentSearchParam(child.getId()), + child, + getNameForContent(child), + displayCount + ); + } + /** * Get display name for the given content. - * + * * @param content The content. - * + * * @return Display name for the content. */ private String getNameForContent(Content content) { if (content instanceof Volume) { - return FileSystemColumnUtils.getVolumeDisplayName((Volume)content); + return FileSystemColumnUtils.getVolumeDisplayName((Volume) content); } else if (content instanceof AbstractFile) { return FileSystemColumnUtils.convertDotDirName((AbstractFile) content); } return content.getName(); } + private TreeResultsDTO.TreeItemDTO createTreeItem(DAOEvent daoEvent, TreeDisplayCount count) { + if (daoEvent instanceof FileSystemContentEvent) { + return createDisplayableContentTreeItem(((FileSystemContentEvent) daoEvent).getContent(), count); + } else if (daoEvent instanceof FileSystemHostEvent) { + // GVDTODO not currently integrated into tree + } else if (daoEvent instanceof FileSystemPersonEvent) { + // GVDTODO not currently integrated into tree + } + + return null; + } + + @Override + void clearCaches() { + this.searchParamsCache.invalidateAll(); + handleIngestComplete(); + } + + @Override + Set handleIngestComplete() { + return getIngestCompleteEvents(this.treeCounts, + (daoEvt, count) -> createTreeItem(daoEvt, count)); + } + + @Override + Set shouldRefreshTree() { + return getRefreshEvents(this.treeCounts, + (daoEvt, count) -> createTreeItem(daoEvt, count)); + } + /** * Handles fetching and paging of data for file types by mime type. */ diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index 26134d71f4..416b55e977 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -723,12 +723,6 @@ public class ViewsDAO extends AbstractDAO { return Pair.of(mimePrefix, mimeSuffix); } - @Override - void clearCaches() { - this.searchParamsCache.invalidateAll(); - handleIngestComplete(); - } - private TreeItemDTO createTreeItem(DAOEvent daoEvent, TreeDisplayCount count) { if (daoEvent instanceof FileTypeExtensionsEvent) { FileTypeExtensionsEvent extEvt = (FileTypeExtensionsEvent) daoEvent; @@ -746,6 +740,12 @@ public class ViewsDAO extends AbstractDAO { } } + @Override + void clearCaches() { + this.searchParamsCache.invalidateAll(); + handleIngestComplete(); + } + @Override Set handleIngestComplete() { return getIngestCompleteEvents(this.treeCounts, diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java index a72c93cea2..fdeefdeb5d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.mainui.datamodel.events; import java.util.Objects; +import org.sleuthkit.datamodel.Content; /** * An event signaling that children files were added or removed from the given @@ -26,26 +27,24 @@ import java.util.Objects; */ public class FileSystemContentEvent implements DAOEvent { - private final Long contentObjectId; + private final Content content; - /** - * Main constructor. - * - * @param contentObjectId The parent content object id. If null, performs - * full refresh of file tree. - */ - public FileSystemContentEvent(Long contentObjectId) { - this.contentObjectId = contentObjectId; + public FileSystemContentEvent(Content content) { + this.content = content; + } + + public Content getContent() { + return content; } public Long getContentObjectId() { - return contentObjectId; + return (content == null) ? null : content.getId(); } @Override public int hashCode() { int hash = 7; - hash = 67 * hash + Objects.hashCode(this.contentObjectId); + hash = 71 * hash + Objects.hashCode(this.content); return hash; } @@ -61,7 +60,7 @@ public class FileSystemContentEvent implements DAOEvent { return false; } final FileSystemContentEvent other = (FileSystemContentEvent) obj; - if (!Objects.equals(this.contentObjectId, other.contentObjectId)) { + if (!Objects.equals(this.content, other.content)) { return false; } return true; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java index 853eb86556..fc632f7b2a 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java @@ -36,6 +36,7 @@ import org.sleuthkit.autopsy.mainui.datamodel.AnalysisResultSetSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.KeywordHitSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; +import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; import static org.sleuthkit.autopsy.mainui.nodes.TreeNode.getDefaultLookup; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -115,8 +116,8 @@ public class AnalysisResultTypeFactory extends TreeChildFactory o1, TreeItemDTO o2) { + return o1.getSearchParams().getArtifactType().getDisplayName().compareTo(o2.getSearchParams().getArtifactType().getDisplayName()); } /** @@ -216,8 +217,8 @@ public class AnalysisResultTypeFactory extends TreeChildFactory o1, TreeItemDTO o2) { + return STRING_COMPARATOR.compare(o1.getSearchParams().getSetName(), o2.getSearchParams().getSetName()); } } @@ -337,8 +338,8 @@ public class AnalysisResultTypeFactory extends TreeChildFactory o1, TreeItemDTO o2) { + return STRING_COMPARATOR.compare(o1.getSearchParams().getSearchTerm(), o2.getSearchParams().getSearchTerm()); } } @@ -439,8 +440,8 @@ public class AnalysisResultTypeFactory extends TreeChildFactory o1, TreeItemDTO o2) { + return STRING_COMPARATOR.compare(o1.getSearchParams().getKeywordMatch(), o2.getSearchParams().getKeywordMatch()); } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java index 4fdb88659d..f8b94b6db5 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/DataArtifactTypeFactory.java @@ -87,9 +87,9 @@ public class DataArtifactTypeFactory extends TreeChildFactory o1, TreeItemDTO o2) { DataArtifactDAO dao = MainDAO.getInstance().getDataArtifactsDAO(); - return dao.getDisplayName(o1.getArtifactType()).compareToIgnoreCase(dao.getDisplayName(o2.getArtifactType())); + return dao.getDisplayName(o1.getSearchParams().getArtifactType()).compareToIgnoreCase(dao.getDisplayName(o2.getSearchParams().getArtifactType())); } private static String getIconPath(BlackboardArtifact.Type artType) { @@ -206,8 +206,8 @@ public class DataArtifactTypeFactory extends TreeChildFactory o1, TreeItemDTO o2) { + return o1.getSearchParams().getType().getDisplayName().compareToIgnoreCase(o2.getSearchParams().getType().getDisplayName()); } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java index e7871526e2..c0b1d3d183 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java @@ -38,6 +38,7 @@ import org.sleuthkit.autopsy.mainui.datamodel.FileSystemColumnUtils; import org.sleuthkit.autopsy.mainui.datamodel.MediaTypeUtils; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; +import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; import static org.sleuthkit.autopsy.mainui.nodes.NodeIconUtil.CARVED_FILE; import static org.sleuthkit.autopsy.mainui.nodes.NodeIconUtil.DELETED_FILE; @@ -139,14 +140,13 @@ public class FileSystemFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent treeEvt) { - // GVDTODO - return null; + // GVDTODO handle virtual directory creation (may be fine as events may invalidate all) + return getTypedTreeItem(treeEvt, FileSystemContentSearchParam.class); } @Override - public int compare(FileSystemContentSearchParam o1, FileSystemContentSearchParam o2) { - // GVDTODO - return 0; + public int compare(TreeItemDTO o1, TreeItemDTO o2) { + return o1.getDisplayName().compareToIgnoreCase(o2.getDisplayName()); } /** @@ -195,14 +195,12 @@ public class FileSystemFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent treeEvt) { - // GVDTODO - return null; + return getTypedTreeItem(treeEvt, FileSystemContentSearchParam.class); } @Override - public int compare(FileSystemContentSearchParam o1, FileSystemContentSearchParam o2) { - // GVDTODO - return 0; + public int compare(TreeItemDTO o1, TreeItemDTO o2) { + return o1.getDisplayName().compareToIgnoreCase(o2.getDisplayName()); } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java index 74baf01e6f..52f945d80d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java @@ -42,7 +42,7 @@ import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; /** * Factory for populating child nodes in a tree based on TreeResultsDTO */ -public abstract class TreeChildFactory extends ChildFactory.Detachable implements Comparator { +public abstract class TreeChildFactory extends ChildFactory.Detachable implements Comparator> { private static final Logger logger = Logger.getLogger(TreeChildFactory.class.getName()); @@ -140,7 +140,8 @@ public abstract class TreeChildFactory extends ChildFactory.Detachable curItem = this.curItemsList.get(insertIndex); + if (this.compare(item, curItem) < 0) { break; } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java index 12bea771b2..b1fb009ae2 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java @@ -36,6 +36,7 @@ import org.sleuthkit.autopsy.mainui.datamodel.FileTypeMimeSearchParams; import org.sleuthkit.autopsy.mainui.datamodel.FileTypeSizeSearchParams; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; +import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; /** @@ -92,8 +93,8 @@ public class ViewsTypeFactory { } @Override - public int compare(FileTypeSizeSearchParams o1, FileTypeSizeSearchParams o2) { - return Integer.compare(o1.getSizeFilter().getId(), o2.getSizeFilter().getId()); + public int compare(TreeItemDTO o1, TreeItemDTO o2) { + return Integer.compare(o1.getSearchParams().getSizeFilter().getId(), o2.getSearchParams().getSizeFilter().getId()); } /** @@ -165,8 +166,8 @@ public class ViewsTypeFactory { } @Override - public int compare(FileTypeMimeSearchParams o1, FileTypeMimeSearchParams o2) { - return STRING_COMPARATOR.compare(o1.getMimeType(), o2.getMimeType()); + public int compare(TreeItemDTO o1, TreeItemDTO o2) { + return STRING_COMPARATOR.compare(o1.getSearchParams().getMimeType(), o2.getSearchParams().getMimeType()); } static class FileMimePrefixNode extends TreeNode { @@ -241,8 +242,8 @@ public class ViewsTypeFactory { } @Override - public int compare(FileTypeMimeSearchParams o1, FileTypeMimeSearchParams o2) { - return STRING_COMPARATOR.compare(o1.getMimeType(), o2.getMimeType()); + public int compare(TreeItemDTO o1, TreeItemDTO o2) { + return STRING_COMPARATOR.compare(o1.getSearchParams().getMimeType(), o2.getSearchParams().getMimeType()); } /** @@ -339,8 +340,8 @@ public class ViewsTypeFactory { } @Override - public int compare(FileTypeExtensionsSearchParams o1, FileTypeExtensionsSearchParams o2) { - return STRING_COMPARATOR.compare(o1.getFilter().getDisplayName(), o2.getFilter().getDisplayName()); + public int compare(TreeItemDTO o1, TreeItemDTO o2) { + return STRING_COMPARATOR.compare(o1.getSearchParams().getFilter().getDisplayName(), o2.getSearchParams().getFilter().getDisplayName()); } /** From dc1a351f2cb2ce874a0c6391671def2bd89af0aa Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 3 Dec 2021 10:05:30 -0500 Subject: [PATCH 13/24] work to fix file system issues --- .../corecomponents/Bundle.properties-MERGED | 6 +- .../mainui/datamodel/FileSystemDAO.java | 159 +++++++++++++++++- .../mainui/datamodel/OsAccountsDAO.java | 1 - .../events/FileSystemContentEvent.java | 17 +- .../FileSystemDataSourceRefreshEvent.java | 65 +++++++ 5 files changed, 235 insertions(+), 13 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED index cde18d3900..468e6d9430 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED @@ -72,9 +72,9 @@ DataContentViewerHex.totalPageLabel.text_1=100 DataContentViewerHex.pageLabel2.text=Page # Product Information panel -LBL_Description=
\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
+LBL_Description=
\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
Format_OperatingSystem_Value={0} version {1} running on {2} -LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
+LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
SortChooser.dialogTitle=Choose Sort Criteria ThumbnailViewChildren.progress.cancelling=(Cancelling) # {0} - file name @@ -97,7 +97,7 @@ DataContentViewerHex.goToPageTextField.text= DataContentViewerHex.goToPageLabel.text=Go to Page: DataResultViewerThumbnail.imagesLabel.text=Images: DataResultViewerThumbnail.imagesRangeLabel.text=- -DataResultViewerThumbnail.filePathLabel.text=\ +DataResultViewerThumbnail.filePathLabel.text=\ \ \ AdvancedConfigurationDialog.cancelButton.text=Cancel DataArtifactContentViewer.waitText=Retrieving and preparing data, please wait... DataArtifactContentViewer.errorText=Error retrieving result diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java index 6c016b16ae..9d0a1e3a2d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java @@ -57,12 +57,14 @@ import org.sleuthkit.autopsy.mainui.datamodel.FileRowDTO.LayoutFileRowDTO; import org.sleuthkit.autopsy.mainui.datamodel.FileRowDTO.SlackFileRowDTO; import org.sleuthkit.autopsy.mainui.datamodel.ContentRowDTO.PoolRowDTO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeDisplayCount; +import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; import org.sleuthkit.autopsy.mainui.datamodel.events.FileSystemContentEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileSystemHostEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.FileSystemPersonEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeCounts; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; +import org.sleuthkit.datamodel.AbstractContent; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; @@ -415,9 +417,25 @@ public class FileSystemDAO extends AbstractDAO { List daoEvents = new ArrayList<>(); if (triggerFullRefresh) { - daoEvents.add(new FileSystemContentEvent(null)); + daoEvents.add(new FileSystemContentEvent(null, null, null)); } else if (affectedContent != null) { - daoEvents.add(new FileSystemContentEvent(affectedContent)); + + Long parentContentId = null; + Long parentHostId = null; + + try { + parentContentId = (affectedContent instanceof AbstractContent) + ? ((AbstractContent) affectedContent).getParentId().orElse(null) + : null; + + parentHostId = (affectedContent instanceof DataSource) + ? ((DataSource) affectedContent).getHost().getHostId() + : null; + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "An error occurred while fetching content id and host id for content with id of: " + affectedContent.getId(), ex); + } + + daoEvents.add(new FileSystemContentEvent(affectedContent, parentContentId, parentHostId)); } if (affectedHost != null) { @@ -429,7 +447,7 @@ public class FileSystemDAO extends AbstractDAO { }); List treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() - .map(daoEvt -> new TreeEvent(createTreeItem(daoEvt, TreeDisplayCount.INDETERMINATE), false)) + .map(daoEvt -> createTreeEvent(daoEvt, TreeDisplayCount.INDETERMINATE, false)) .collect(Collectors.toList()); return Stream.of(daoEvents, treeEvents) @@ -571,6 +589,24 @@ public class FileSystemDAO extends AbstractDAO { return content.getName(); } + private TreeEvent createTreeEvent(DAOEvent daoEvent, TreeDisplayCount count, boolean fullRefresh) { + + if (daoEvent instanceof FileSystemContentEvent) { + FileSystemContentEvent contentEvt = (FileSystemContentEvent) daoEvent; + return new FileSystemTreeEvent( + contentEvt.getParentObjId(), + contentEvt.getParentHostId(), + createDisplayableContentTreeItem(contentEvt.getContent(), count), + fullRefresh); + } else if (daoEvent instanceof FileSystemHostEvent) { + // GVDTODO not currently integrated into tree + } else if (daoEvent instanceof FileSystemPersonEvent) { + // GVDTODO not currently integrated into tree + } + + return null; + } + private TreeResultsDTO.TreeItemDTO createTreeItem(DAOEvent daoEvent, TreeDisplayCount count) { if (daoEvent instanceof FileSystemContentEvent) { return createDisplayableContentTreeItem(((FileSystemContentEvent) daoEvent).getContent(), count); @@ -578,8 +614,8 @@ public class FileSystemDAO extends AbstractDAO { // GVDTODO not currently integrated into tree } else if (daoEvent instanceof FileSystemPersonEvent) { // GVDTODO not currently integrated into tree - } - + } + return null; } @@ -591,14 +627,123 @@ public class FileSystemDAO extends AbstractDAO { @Override Set handleIngestComplete() { + return treeCounts.flushEvents().stream() + .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) + .collect(Collectors.toSet()); + return getIngestCompleteEvents(this.treeCounts, (daoEvt, count) -> createTreeItem(daoEvt, count)); } @Override Set shouldRefreshTree() { - return getRefreshEvents(this.treeCounts, - (daoEvt, count) -> createTreeItem(daoEvt, count)); + return treeCounts.getEventTimeouts().stream() + .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) + .collect(Collectors.toSet()); + } + + + + public static class DataSourceRefreshTreeEvent extends TreeEvent { + private final long dataSourceId; + + public DataSourceRefreshTreeEvent(long dataSourceId) { + super(null, true); + this.dataSourceId = dataSourceId; + } + + public long getDataSourceId() { + return dataSourceId; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 37 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final DataSourceRefreshTreeEvent other = (DataSourceRefreshTreeEvent) obj; + if (this.dataSourceId != other.dataSourceId) { + return false; + } + return true; + } + } + + public static class FileSystemTreeEvent extends TreeEvent { + + private final Long parentContentId; + private final Long parentHostId; + private final TreeResultsDTO.TreeItemDTO itemRecord; + + + FileSystemTreeEvent(Long parentContentId, Long parentHostId, TreeItemDTO itemRecord, boolean refreshRequired) { + super(itemRecord, refreshRequired); + this.parentContentId = parentContentId; + this.parentHostId = parentHostId; + this.itemRecord = itemRecord; + } + + public Long getParentContentId() { + return parentContentId; + } + + public Long getParentHostId() { + return parentHostId; + } + + @Override + public TreeResultsDTO.TreeItemDTO getItemRecord() { + // override to be typed to FileSystemContentSearchParam + return itemRecord; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 59 * hash + Objects.hashCode(this.parentContentId); + hash = 59 * hash + Objects.hashCode(this.parentHostId); + hash = 59 * hash + Objects.hashCode(this.itemRecord); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final FileSystemTreeEvent other = (FileSystemTreeEvent) obj; + if (!Objects.equals(this.parentContentId, other.parentContentId)) { + return false; + } + if (!Objects.equals(this.parentHostId, other.parentHostId)) { + return false; + } + if (!Objects.equals(this.itemRecord, other.itemRecord)) { + return false; + } + return true; + } + + } /** diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java index e86ad03f96..ceace7b7f1 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/OsAccountsDAO.java @@ -24,7 +24,6 @@ import com.google.common.cache.CacheBuilder; import java.beans.PropertyChangeEvent; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java index fdeefdeb5d..d44401f106 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java @@ -28,9 +28,21 @@ import org.sleuthkit.datamodel.Content; public class FileSystemContentEvent implements DAOEvent { private final Content content; + private final Long parentObjId; + private final Long parentHostId; - public FileSystemContentEvent(Content content) { + public FileSystemContentEvent(Content content, Long parentObjId, Long parentHostId) { this.content = content; + this.parentObjId = parentObjId; + this.parentHostId = parentHostId; + } + + public Long getParentObjId() { + return parentObjId; + } + + public Long getParentHostId() { + return parentHostId; } public Content getContent() { @@ -40,7 +52,8 @@ public class FileSystemContentEvent implements DAOEvent { public Long getContentObjectId() { return (content == null) ? null : content.getId(); } - + + @Override public int hashCode() { int hash = 7; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java new file mode 100644 index 0000000000..0a0d76d88d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java @@ -0,0 +1,65 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.mainui.datamodel.events; + +/** + * An event signaling that all items from a data source should be refreshed. + */ +public class FileSystemDataSourceRefreshEvent implements DAOEvent { + + private final long dataSourceId; + + public FileSystemDataSourceRefreshEvent(long dataSourceId) { + this.dataSourceId = dataSourceId; + } + + public long getDataSourceId() { + return dataSourceId; + } + + @Override + public int hashCode() { + int hash = 3; + hash = 29 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final FileSystemDataSourceRefreshEvent other = (FileSystemDataSourceRefreshEvent) obj; + if (this.dataSourceId != other.dataSourceId) { + return false; + } + return true; + } + + @Override + public Type getType() { + return Type.RESULT; + } +} From 26ed33f4fb8d4e41aa5fe947518c73b0a1bb9131 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 3 Dec 2021 12:12:58 -0500 Subject: [PATCH 14/24] file system fixes --- .../mainui/datamodel/FileSystemDAO.java | 103 ++++-------------- .../events/FileSystemContentEvent.java | 18 +-- .../FileSystemDataSourceRefreshEvent.java | 65 ----------- .../mainui/nodes/FileSystemFactory.java | 29 ++++- 4 files changed, 60 insertions(+), 155 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java index 9d0a1e3a2d..fdceacacc2 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java @@ -419,23 +419,22 @@ public class FileSystemDAO extends AbstractDAO { if (triggerFullRefresh) { daoEvents.add(new FileSystemContentEvent(null, null, null)); } else if (affectedContent != null) { - Long parentContentId = null; - Long parentHostId = null; + Host parentHost = null; try { parentContentId = (affectedContent instanceof AbstractContent) ? ((AbstractContent) affectedContent).getParentId().orElse(null) : null; - parentHostId = (affectedContent instanceof DataSource) - ? ((DataSource) affectedContent).getHost().getHostId() + parentHost = (affectedContent instanceof DataSource) + ? ((DataSource) affectedContent).getHost() : null; } catch (TskCoreException ex) { logger.log(Level.WARNING, "An error occurred while fetching content id and host id for content with id of: " + affectedContent.getId(), ex); } - daoEvents.add(new FileSystemContentEvent(affectedContent, parentContentId, parentHostId)); + daoEvents.add(new FileSystemContentEvent(affectedContent, parentContentId, parentHost)); } if (affectedHost != null) { @@ -566,9 +565,9 @@ public class FileSystemDAO extends AbstractDAO { private TreeResultsDTO.TreeItemDTO createDisplayableContentTreeItem(Content child, TreeDisplayCount displayCount) { return new TreeResultsDTO.TreeItemDTO<>( FileSystemContentSearchParam.getTypeId(), - new FileSystemContentSearchParam(child.getId()), + new FileSystemContentSearchParam(child == null ? null : child.getId()), child, - getNameForContent(child), + child == null ? null : getNameForContent(child), displayCount ); } @@ -594,27 +593,15 @@ public class FileSystemDAO extends AbstractDAO { if (daoEvent instanceof FileSystemContentEvent) { FileSystemContentEvent contentEvt = (FileSystemContentEvent) daoEvent; return new FileSystemTreeEvent( - contentEvt.getParentObjId(), - contentEvt.getParentHostId(), - createDisplayableContentTreeItem(contentEvt.getContent(), count), + contentEvt.getParentObjId(), + contentEvt.getParentHost(), + createDisplayableContentTreeItem(contentEvt.getContent(), count), fullRefresh); } else if (daoEvent instanceof FileSystemHostEvent) { // GVDTODO not currently integrated into tree } else if (daoEvent instanceof FileSystemPersonEvent) { // GVDTODO not currently integrated into tree } - - return null; - } - - private TreeResultsDTO.TreeItemDTO createTreeItem(DAOEvent daoEvent, TreeDisplayCount count) { - if (daoEvent instanceof FileSystemContentEvent) { - return createDisplayableContentTreeItem(((FileSystemContentEvent) daoEvent).getContent(), count); - } else if (daoEvent instanceof FileSystemHostEvent) { - // GVDTODO not currently integrated into tree - } else if (daoEvent instanceof FileSystemPersonEvent) { - // GVDTODO not currently integrated into tree - } return null; } @@ -627,72 +614,36 @@ public class FileSystemDAO extends AbstractDAO { @Override Set handleIngestComplete() { - return treeCounts.flushEvents().stream() - .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) + return treeCounts.flushEvents().stream() + .map(daoEvt -> createTreeEvent(daoEvt, TreeDisplayCount.UNSPECIFIED, true)) .collect(Collectors.toSet()); - - return getIngestCompleteEvents(this.treeCounts, - (daoEvt, count) -> createTreeItem(daoEvt, count)); } @Override Set shouldRefreshTree() { - return treeCounts.getEventTimeouts().stream() - .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) + return treeCounts.getEventTimeouts().stream() + .map(daoEvt -> createTreeEvent(daoEvt, TreeDisplayCount.UNSPECIFIED, true)) .collect(Collectors.toSet()); } - - - + public static class DataSourceRefreshTreeEvent extends TreeEvent { - private final long dataSourceId; - public DataSourceRefreshTreeEvent(long dataSourceId) { - super(null, true); - this.dataSourceId = dataSourceId; + public DataSourceRefreshTreeEvent(boolean refresh) { + super(null, refresh); } - public long getDataSourceId() { - return dataSourceId; - } - - @Override - public int hashCode() { - int hash = 7; - hash = 37 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); - return hash; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final DataSourceRefreshTreeEvent other = (DataSourceRefreshTreeEvent) obj; - if (this.dataSourceId != other.dataSourceId) { - return false; - } - return true; - } } public static class FileSystemTreeEvent extends TreeEvent { private final Long parentContentId; - private final Long parentHostId; + private final Host parentHost; private final TreeResultsDTO.TreeItemDTO itemRecord; - - FileSystemTreeEvent(Long parentContentId, Long parentHostId, TreeItemDTO itemRecord, boolean refreshRequired) { + FileSystemTreeEvent(Long parentContentId, Host parentHost, TreeItemDTO itemRecord, boolean refreshRequired) { super(itemRecord, refreshRequired); this.parentContentId = parentContentId; - this.parentHostId = parentHostId; + this.parentHost = parentHost; this.itemRecord = itemRecord; } @@ -700,8 +651,8 @@ public class FileSystemDAO extends AbstractDAO { return parentContentId; } - public Long getParentHostId() { - return parentHostId; + public Host getParentHost() { + return parentHost; } @Override @@ -712,10 +663,8 @@ public class FileSystemDAO extends AbstractDAO { @Override public int hashCode() { - int hash = 7; - hash = 59 * hash + Objects.hashCode(this.parentContentId); - hash = 59 * hash + Objects.hashCode(this.parentHostId); - hash = 59 * hash + Objects.hashCode(this.itemRecord); + int hash = 5; + hash = 31 * hash + Objects.hashCode(this.itemRecord); return hash; } @@ -731,12 +680,6 @@ public class FileSystemDAO extends AbstractDAO { return false; } final FileSystemTreeEvent other = (FileSystemTreeEvent) obj; - if (!Objects.equals(this.parentContentId, other.parentContentId)) { - return false; - } - if (!Objects.equals(this.parentHostId, other.parentHostId)) { - return false; - } if (!Objects.equals(this.itemRecord, other.itemRecord)) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java index d44401f106..8efc3d5ef9 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemContentEvent.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.mainui.datamodel.events; import java.util.Objects; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.Host; /** * An event signaling that children files were added or removed from the given @@ -29,22 +30,26 @@ public class FileSystemContentEvent implements DAOEvent { private final Content content; private final Long parentObjId; - private final Long parentHostId; + private final Host parentHost; - public FileSystemContentEvent(Content content, Long parentObjId, Long parentHostId) { + public FileSystemContentEvent(Content content, Long parentObjId, Host parentHost) { this.content = content; this.parentObjId = parentObjId; - this.parentHostId = parentHostId; + this.parentHost = parentHost; } public Long getParentObjId() { return parentObjId; } - public Long getParentHostId() { - return parentHostId; + public Host getParentHost() { + return parentHost; } + /** + * @return The content associated with the event, if null, triggers a full + * refresh. + */ public Content getContent() { return content; } @@ -52,8 +57,7 @@ public class FileSystemContentEvent implements DAOEvent { public Long getContentObjectId() { return (content == null) ? null : content.getId(); } - - + @Override public int hashCode() { int hash = 7; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java deleted file mode 100644 index 0a0d76d88d..0000000000 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileSystemDataSourceRefreshEvent.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2021 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.mainui.datamodel.events; - -/** - * An event signaling that all items from a data source should be refreshed. - */ -public class FileSystemDataSourceRefreshEvent implements DAOEvent { - - private final long dataSourceId; - - public FileSystemDataSourceRefreshEvent(long dataSourceId) { - this.dataSourceId = dataSourceId; - } - - public long getDataSourceId() { - return dataSourceId; - } - - @Override - public int hashCode() { - int hash = 3; - hash = 29 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); - return hash; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final FileSystemDataSourceRefreshEvent other = (FileSystemDataSourceRefreshEvent) obj; - if (this.dataSourceId != other.dataSourceId) { - return false; - } - return true; - } - - @Override - public Type getType() { - return Type.RESULT; - } -} diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java index c54d535bf1..3936451ae0 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.mainui.nodes; +import java.util.Objects; import java.util.Optional; import org.openide.nodes.Children; import org.openide.nodes.Node; @@ -35,6 +36,7 @@ import org.sleuthkit.autopsy.directorytree.ExtractUnallocAction; import org.sleuthkit.autopsy.directorytree.FileSystemDetailsAction; import org.sleuthkit.autopsy.mainui.datamodel.FileSystemContentSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.FileSystemColumnUtils; +import org.sleuthkit.autopsy.mainui.datamodel.FileSystemDAO.FileSystemTreeEvent; import org.sleuthkit.autopsy.mainui.datamodel.MediaTypeUtils; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; @@ -140,8 +142,18 @@ public class FileSystemFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent treeEvt) { - // GVDTODO handle virtual directory creation (may be fine as events may invalidate all) - return getTypedTreeItem(treeEvt, FileSystemContentSearchParam.class); + if (treeEvt instanceof FileSystemTreeEvent) { + FileSystemTreeEvent fsTreeEvent = (FileSystemTreeEvent) treeEvt; + // when getContentObjectId == null, trigger refresh, otherwise, see if common parent + if (fsTreeEvent.getItemRecord().getSearchParams().getContentObjectId() == null + || (Objects.equals(this.host, fsTreeEvent.getParentHost()) + && Objects.equals(this.contentId, fsTreeEvent.getParentContentId()))) { + + return fsTreeEvent.getItemRecord(); + } + } + + return null; } @Override @@ -195,7 +207,18 @@ public class FileSystemFactory extends TreeChildFactory getOrCreateRelevantChild(TreeEvent treeEvt) { - return getTypedTreeItem(treeEvt, FileSystemContentSearchParam.class); + if (treeEvt instanceof FileSystemTreeEvent) { + FileSystemTreeEvent fsTreeEvent = (FileSystemTreeEvent) treeEvt; + // when getContentObjectId == null, trigger refresh, otherwise, see if common parent + if (fsTreeEvent.getItemRecord().getSearchParams().getContentObjectId() == null + || Objects.equals(fsTreeEvent.getItemRecord().getSearchParams().getContentObjectId(), dataSourceId)) { + + return fsTreeEvent.getItemRecord(); + } + } + + return null; + } @Override From 6b5b560538dd3b999745f8d3c01a579a42482136 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 3 Dec 2021 13:34:18 -0500 Subject: [PATCH 15/24] fixes --- .../autopsy/mainui/datamodel/AbstractDAO.java | 4 +++- .../sleuthkit/autopsy/mainui/datamodel/MainDAO.java | 3 +++ .../autopsy/mainui/nodes/ViewsTypeFactory.java | 13 ++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java index 3123dccc62..5542dbd9d7 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AbstractDAO.java @@ -100,7 +100,7 @@ abstract class AbstractDAO { * Determines what keys should be kept in the cache while iterating through * all the keys. * - * @param cache The cache. + * @param cache The cache. * @param shouldInvalidate If the key should be removed from the cache. */ static void invalidateKeys(Cache, ?> cache, Predicate shouldInvalidate) { @@ -125,6 +125,7 @@ abstract class AbstractDAO { static Set getIngestCompleteEvents(TreeCounts treeCounts, BiFunction> converter) { return treeCounts.flushEvents().stream() .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) + .filter(evt -> evt != null) .collect(Collectors.toSet()); } @@ -141,6 +142,7 @@ abstract class AbstractDAO { static Set getRefreshEvents(TreeCounts treeCounts, BiFunction> converter) { return treeCounts.getEventTimeouts().stream() .map(daoEvt -> new TreeEvent(converter.apply(daoEvt, TreeDisplayCount.UNSPECIFIED), true)) + .filter(evt -> evt != null) .collect(Collectors.toSet()); } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java index 44de97b935..817a3e212c 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java @@ -259,6 +259,7 @@ public class MainDAO extends AbstractDAO { return allDAOs.stream() .map(subDAO -> subDAO.processEvent(evt)) .flatMap(evts -> evts == null ? Stream.empty() : evts.stream()) + .filter(e -> e != null) .collect(Collectors.toSet()); } @@ -267,6 +268,7 @@ public class MainDAO extends AbstractDAO { return allDAOs.stream() .map((subDAO) -> subDAO.shouldRefreshTree()) .flatMap(evts -> evts == null ? Stream.empty() : evts.stream()) + .filter(e -> e != null) .collect(Collectors.toSet()); } @@ -280,6 +282,7 @@ public class MainDAO extends AbstractDAO { return daoStreamEvts.stream() .flatMap(evts -> evts == null ? Stream.empty() : evts.stream()) + .filter(evt -> evt != null) .collect(Collectors.toSet()); } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java index b1fb009ae2..e59903225b 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java @@ -154,11 +154,17 @@ public class ViewsTypeFactory { // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. FileTypeMimeSearchParams searchParam = originalTreeItem.getSearchParams(); + String mimePrefix = searchParam.getMimeType() == null ? "" : searchParam.getMimeType(); + int indexOfSlash = mimePrefix.indexOf("/"); + if (indexOfSlash >= 0) { + mimePrefix = mimePrefix.substring(0, indexOfSlash); + } + return new TreeResultsDTO.TreeItemDTO<>( AnalysisResultSearchParam.getTypeId(), - new FileTypeMimeSearchParams(searchParam.getMimeType(), this.dataSourceId), - searchParam.getMimeType(), - searchParam.getMimeType(), + new FileTypeMimeSearchParams(mimePrefix, this.dataSourceId), + mimePrefix, + mimePrefix, originalTreeItem.getDisplayCount()); } return null; @@ -325,6 +331,7 @@ public class ViewsTypeFactory { TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, FileTypeExtensionsSearchParams.class); if (originalTreeItem != null + && this.childFilters.contains(originalTreeItem.getSearchParams().getFilter()) && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. From e64c93d0860a22f3a26befee253a91f0fd2bf72e Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 3 Dec 2021 15:21:59 -0500 Subject: [PATCH 16/24] fixes --- .../mainui/datamodel/FileSystemDAO.java | 51 ++++++++++++++----- .../mainui/nodes/FileSystemFactory.java | 8 +++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java index fdceacacc2..a112afae6d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java @@ -359,7 +359,7 @@ public class FileSystemDAO extends AbstractDAO { @Override Set processEvent(PropertyChangeEvent evt) { - Content affectedParentContent = null; + Content affectedContent = null; Host affectedParentHost = null; // GVDTODO person parents and parent of persons not handled yet @@ -381,7 +381,7 @@ public class FileSystemDAO extends AbstractDAO { if (invalidatesAllFileSystem(parentContent)) { refreshAllContent = true; } else { - affectedParentContent = parentContent; + affectedContent = content; } } else if (evt instanceof DataSourceAddedEvent) { Host host = getHostFromDs(((DataSourceAddedEvent) evt).getDataSource()); @@ -404,13 +404,13 @@ public class FileSystemDAO extends AbstractDAO { } // if nothing affected, return no events - if (!refreshAllContent && affectedParentContent == null && affectedParentHost == null && !affectedParentPerson.isPresent()) { + if (!refreshAllContent && affectedContent == null && affectedParentHost == null && !affectedParentPerson.isPresent()) { return Collections.emptySet(); } - invalidateKeys(affectedParentPerson, affectedParentHost, affectedParentContent, refreshAllContent); + invalidateKeys(affectedParentPerson, affectedParentHost, affectedContent, refreshAllContent); - return getDAOEvents(affectedParentPerson, affectedParentHost, affectedParentContent, refreshAllContent); + return getDAOEvents(affectedParentPerson, affectedParentHost, affectedContent, refreshAllContent); } private Set getDAOEvents(Optional affectedPerson, Host affectedHost, Content affectedContent, boolean triggerFullRefresh) { @@ -447,6 +447,7 @@ public class FileSystemDAO extends AbstractDAO { List treeEvents = this.treeCounts.enqueueAll(daoEvents).stream() .map(daoEvt -> createTreeEvent(daoEvt, TreeDisplayCount.INDETERMINATE, false)) + .filter(evt -> evt != null) .collect(Collectors.toList()); return Stream.of(daoEvents, treeEvents) @@ -562,12 +563,13 @@ public class FileSystemDAO extends AbstractDAO { } } - private TreeResultsDTO.TreeItemDTO createDisplayableContentTreeItem(Content child, TreeDisplayCount displayCount) { - return new TreeResultsDTO.TreeItemDTO<>( + private FileSystemTreeItem createDisplayableContentTreeItem(Content child, TreeDisplayCount displayCount) { + return new FileSystemTreeItem( FileSystemContentSearchParam.getTypeId(), new FileSystemContentSearchParam(child == null ? null : child.getId()), child, child == null ? null : getNameForContent(child), + child instanceof AbstractFile ? ((AbstractFile) child).getMetaType() : null, displayCount ); } @@ -592,11 +594,13 @@ public class FileSystemDAO extends AbstractDAO { if (daoEvent instanceof FileSystemContentEvent) { FileSystemContentEvent contentEvt = (FileSystemContentEvent) daoEvent; + return new FileSystemTreeEvent( contentEvt.getParentObjId(), contentEvt.getParentHost(), createDisplayableContentTreeItem(contentEvt.getContent(), count), fullRefresh); + } else if (daoEvent instanceof FileSystemHostEvent) { // GVDTODO not currently integrated into tree } else if (daoEvent instanceof FileSystemPersonEvent) { @@ -616,6 +620,7 @@ public class FileSystemDAO extends AbstractDAO { Set handleIngestComplete() { return treeCounts.flushEvents().stream() .map(daoEvt -> createTreeEvent(daoEvt, TreeDisplayCount.UNSPECIFIED, true)) + .filter(evt -> evt != null) .collect(Collectors.toSet()); } @@ -623,6 +628,7 @@ public class FileSystemDAO extends AbstractDAO { Set shouldRefreshTree() { return treeCounts.getEventTimeouts().stream() .map(daoEvt -> createTreeEvent(daoEvt, TreeDisplayCount.UNSPECIFIED, true)) + .filter(evt -> evt != null) .collect(Collectors.toSet()); } @@ -634,13 +640,35 @@ public class FileSystemDAO extends AbstractDAO { } + public static class FileSystemTreeItem extends TreeItemDTO { + + private final TskData.TSK_FS_META_TYPE_ENUM metaType; + + FileSystemTreeItem( + String typeId, + FileSystemContentSearchParam searchParams, + Object id, + String displayName, + TskData.TSK_FS_META_TYPE_ENUM metaType, + TreeDisplayCount count) { + + super(typeId, searchParams, id, displayName, count); + this.metaType = metaType; + } + + public TskData.TSK_FS_META_TYPE_ENUM getMetaType() { + return metaType; + } + + } + public static class FileSystemTreeEvent extends TreeEvent { private final Long parentContentId; private final Host parentHost; - private final TreeResultsDTO.TreeItemDTO itemRecord; + private final FileSystemTreeItem itemRecord; - FileSystemTreeEvent(Long parentContentId, Host parentHost, TreeItemDTO itemRecord, boolean refreshRequired) { + FileSystemTreeEvent(Long parentContentId, Host parentHost, FileSystemTreeItem itemRecord, boolean refreshRequired) { super(itemRecord, refreshRequired); this.parentContentId = parentContentId; this.parentHost = parentHost; @@ -656,8 +684,8 @@ public class FileSystemDAO extends AbstractDAO { } @Override - public TreeResultsDTO.TreeItemDTO getItemRecord() { - // override to be typed to FileSystemContentSearchParam + public FileSystemTreeItem getItemRecord() { + // override to be typed and contain extra information return itemRecord; } @@ -686,7 +714,6 @@ public class FileSystemDAO extends AbstractDAO { return true; } - } /** diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java index 3936451ae0..2541ec8d27 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java @@ -37,6 +37,7 @@ import org.sleuthkit.autopsy.directorytree.FileSystemDetailsAction; import org.sleuthkit.autopsy.mainui.datamodel.FileSystemContentSearchParam; import org.sleuthkit.autopsy.mainui.datamodel.FileSystemColumnUtils; import org.sleuthkit.autopsy.mainui.datamodel.FileSystemDAO.FileSystemTreeEvent; +import org.sleuthkit.autopsy.mainui.datamodel.FileSystemDAO.FileSystemTreeItem; import org.sleuthkit.autopsy.mainui.datamodel.MediaTypeUtils; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; @@ -158,6 +159,13 @@ public class FileSystemFactory extends TreeChildFactory o1, TreeItemDTO o2) { + if (o1 instanceof FileSystemTreeItem && o2 instanceof FileSystemTreeItem) { + FileSystemTreeItem fs1 = (FileSystemTreeItem) o1; + FileSystemTreeItem fs2 = (FileSystemTreeItem) o2; + if (fs1.getMetaType().getValue() != fs2.getMetaType().getValue()) { + return Short.compare(fs1.getMetaType().getValue(), fs2.getMetaType().getValue()); + } + } return o1.getDisplayName().compareToIgnoreCase(o2.getDisplayName()); } From bdc497e705cb17d792fc4e355792be9365005a79 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 3 Dec 2021 15:26:05 -0500 Subject: [PATCH 17/24] fix --- .../org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java index 2541ec8d27..4044efa4dc 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/FileSystemFactory.java @@ -162,8 +162,9 @@ public class FileSystemFactory extends TreeChildFactory Date: Mon, 6 Dec 2021 09:35:46 -0500 Subject: [PATCH 18/24] synchronized fix --- .../mainui/nodes/TreeChildFactory.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java index 74baf01e6f..617f400624 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java @@ -86,20 +86,21 @@ public abstract class TreeChildFactory extends ChildFactory.Detachable toPopulate) { List> itemsList; - synchronized (resultsUpdateLock) { - // Load data from DAO if we haven't already - if (curResults == null) { - try { - updateData(); - } catch (IllegalArgumentException | ExecutionException ex) { - logger.log(Level.WARNING, "An error occurred while fetching keys", ex); - return false; - } - } - // make copy to avoid concurrent modification - itemsList = new ArrayList<>(curItemsList); - } + // Load data from DAO if we haven't already + if (curResults == null) { + try { + updateData(); + } catch (IllegalArgumentException | ExecutionException ex) { + logger.log(Level.WARNING, "An error occurred while fetching keys", ex); + return false; + } + } + // make copy to avoid concurrent modification + synchronized (resultsUpdateLock) { + itemsList = new ArrayList<>(curItemsList); + } + // update existing cached nodes List curResultIds = new ArrayList<>(); for (TreeItemDTO dto : itemsList) { @@ -159,8 +160,9 @@ public abstract class TreeChildFactory extends ChildFactory.Detachable newResults = getChildResults(); synchronized (resultsUpdateLock) { - this.curResults = getChildResults(); + this.curResults = newResults; Map> idMapping = new HashMap<>(); List> curItemsList = new ArrayList<>(); for (TreeItemDTO item : this.curResults.getItems()) { From 92b94e67b6b08c70ba7de7a0438ca90cfe15d4ce Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 6 Dec 2021 10:23:08 -0500 Subject: [PATCH 19/24] integrate views base filters --- .../autopsy/mainui/datamodel/ViewsDAO.java | 70 ++++++++++++++----- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index b6434352be..9b5730a0c1 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.mainui.datamodel; import org.sleuthkit.autopsy.mainui.datamodel.events.DAOEvent; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableSet; import java.beans.PropertyChangeEvent; import java.sql.SQLException; import java.util.ArrayList; @@ -33,7 +34,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Set; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; @@ -62,6 +62,7 @@ import org.sleuthkit.datamodel.CaseDbAccessManager.CaseDbPreparedStatement; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; +import org.sleuthkit.datamodel.TskData.TSK_FS_NAME_TYPE_ENUM; /** * Provides information to populate the results viewer for data in the views @@ -195,6 +196,22 @@ public class ViewsDAO extends AbstractDAO { .collect(Collectors.joining(", ")) + ")"; } + /** + * @return If user preference of hide known files, returns sql and clause to + * hide known files or returns empty string otherwise. + */ + private String getHideKnownAndClause() { + return (hideKnownFilesInViewsTree() ? (" AND (known IS NULL OR known <> " + TskData.FileKnown.KNOWN.getFileKnownValue() + ") ") : ""); + } + + /** + * @return A clause (no 'and' or 'where' prefixed) indicating the dir_type + * is regular. + */ + private String getRegDirTypeClause() { + return "(dir_type = " + TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue() + ")"; + } + /** * Returns a clause that will filter out files that aren't to be counted in * the file extensions view. @@ -202,8 +219,7 @@ public class ViewsDAO extends AbstractDAO { * @return The filter that will need to be proceeded with 'where' or 'and'. */ private String getBaseFileExtensionFilter() { - return "(dir_type = " + TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue() + ")" - + (hideKnownFilesInViewsTree() ? (" AND (known IS NULL OR known <> " + TskData.FileKnown.KNOWN.getFileKnownValue() + ")") : ""); + return getRegDirTypeClause() + getHideKnownAndClause(); } /** @@ -223,6 +239,22 @@ public class ViewsDAO extends AbstractDAO { return whereClause; } + /** + * @return The TSK_DB_FILES_TYPE_ENUm values allowed for mime type view + * items. + */ + private Set getMimeDbFilesTypes() { + return Stream.of( + TskData.TSK_DB_FILES_TYPE_ENUM.FS, + TskData.TSK_DB_FILES_TYPE_ENUM.CARVED, + TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED, + TskData.TSK_DB_FILES_TYPE_ENUM.LAYOUT_FILE, + TskData.TSK_DB_FILES_TYPE_ENUM.LOCAL, + (hideSlackFilesInViewsTree() ? null : (TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) + .filter(ordinal -> ordinal != null) + .collect(Collectors.toSet()); + } + /** * Returns a statement to be proceeded with 'where' or 'and' that will * filter out results that should not be viewed in mime types view. @@ -230,15 +262,10 @@ public class ViewsDAO extends AbstractDAO { * @return A statement to be proceeded with 'and' or 'where'. */ private String getBaseFileMimeFilter() { - return "(dir_type = " + TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue() + ")" - + (hideKnownFilesInViewsTree() ? (" AND (known IS NULL OR known != " + TskData.FileKnown.KNOWN.getFileKnownValue() + ")") : "") + return getRegDirTypeClause() + + getHideKnownAndClause() + " AND (type IN (" - + TskData.TSK_DB_FILES_TYPE_ENUM.FS.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.CARVED.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.LAYOUT_FILE.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.LOCAL.ordinal() - + (hideSlackFilesInViewsTree() ? "" : ("," + TskData.TSK_DB_FILES_TYPE_ENUM.SLACK.ordinal())) + + getMimeDbFilesTypes().stream().map(v -> Integer.toString(v.ordinal())).collect(Collectors.joining(", ")) + "))"; } @@ -281,8 +308,7 @@ public class ViewsDAO extends AbstractDAO { */ private String getBaseFileSizeFilter() { // Ignore unallocated block files. - return "(type != " + TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.getFileType() + ")" - + ((hideKnownFilesInViewsTree() ? (" AND (known IS NULL OR known != " + TskData.FileKnown.KNOWN.getFileKnownValue() + ")") : "")); //NON-NLS + return "(type != " + TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.getFileType() + ")" + getHideKnownAndClause(); } /** @@ -763,23 +789,29 @@ public class ViewsDAO extends AbstractDAO { AbstractFile af = DAOEventUtils.getFileFromFileEvent(evt); if (af == null) { return Collections.emptySet(); + } else if (hideKnownFilesInViewsTree() && TskData.FileKnown.KNOWN.equals(af.getKnown())) { + return Collections.emptySet(); } long dsId = af.getDataSourceObjectId(); // create an extension mapping if extension present - Set evtExtFilters = StringUtils.isBlank(af.getNameExtension()) + Set evtExtFilters = (StringUtils.isBlank(af.getNameExtension()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) ? Collections.emptySet() : EXTENSION_FILTER_MAP.getOrDefault("." + af.getNameExtension(), Collections.emptySet()); // create a mime type mapping if mime type present - String evtMimeType = StringUtils.isBlank(af.getMIMEType()) ? null : af.getMIMEType(); + String evtMimeType = (StringUtils.isBlank(af.getMIMEType()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) || !getMimeDbFilesTypes().contains(af.getType()) + ? null + : af.getMIMEType(); // create a size mapping if size present in filters - FileSizeFilter evtFileSize = Stream.of(FileSizeFilter.values()) - .filter(filter -> af.getSize() >= filter.getMinBound() && (filter.getMaxBound() == null || af.getSize() < filter.getMaxBound())) - .findFirst() - .orElse(null); + FileSizeFilter evtFileSize = !TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType()) + ? null + : Stream.of(FileSizeFilter.values()) + .filter(filter -> af.getSize() >= filter.getMinBound() && (filter.getMaxBound() == null || af.getSize() < filter.getMaxBound())) + .findFirst() + .orElse(null); if (evtExtFilters.isEmpty() && evtMimeType == null && evtFileSize == null) { return Collections.emptySet(); From ce7ca755efb701159a550d45a9a6c244a6fded34 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 6 Dec 2021 11:06:04 -0500 Subject: [PATCH 20/24] fixes --- Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java | 2 +- .../autopsy/mainui/nodes/AnalysisResultTypeFactory.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index 9b5730a0c1..f736313298 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -806,7 +806,7 @@ public class ViewsDAO extends AbstractDAO { : af.getMIMEType(); // create a size mapping if size present in filters - FileSizeFilter evtFileSize = !TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType()) + FileSizeFilter evtFileSize = TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType()) ? null : Stream.of(FileSizeFilter.values()) .filter(filter -> af.getSize() >= filter.getMinBound() && (filter.getMaxBound() == null || af.getSize() < filter.getMaxBound())) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java index fc632f7b2a..1bfb0a28eb 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/AnalysisResultTypeFactory.java @@ -52,7 +52,8 @@ public class AnalysisResultTypeFactory extends TreeChildFactory SET_TREE_ARTIFACTS = ImmutableSet.of( BlackboardArtifact.Type.TSK_HASHSET_HIT.getTypeID(), BlackboardArtifact.Type.TSK_INTERESTING_ARTIFACT_HIT.getTypeID(), - BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT.getTypeID() + BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT.getTypeID(), + BlackboardArtifact.Type.TSK_INTERESTING_ITEM.getTypeID() ); /** From a482765bac144486b6717a0a2b4962c3e60dcf7d Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 6 Dec 2021 15:03:45 -0500 Subject: [PATCH 21/24] fix for module data content --- .../autopsy/mainui/datamodel/FileSystemDAO.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java index a112afae6d..a1b0b48f24 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileSystemDAO.java @@ -360,6 +360,7 @@ public class FileSystemDAO extends AbstractDAO { @Override Set processEvent(PropertyChangeEvent evt) { Content affectedContent = null; + Content affectedParentContent = null; Host affectedParentHost = null; // GVDTODO person parents and parent of persons not handled yet @@ -382,6 +383,7 @@ public class FileSystemDAO extends AbstractDAO { refreshAllContent = true; } else { affectedContent = content; + affectedParentContent = parentContent; } } else if (evt instanceof DataSourceAddedEvent) { Host host = getHostFromDs(((DataSourceAddedEvent) evt).getDataSource()); @@ -410,23 +412,18 @@ public class FileSystemDAO extends AbstractDAO { invalidateKeys(affectedParentPerson, affectedParentHost, affectedContent, refreshAllContent); - return getDAOEvents(affectedParentPerson, affectedParentHost, affectedContent, refreshAllContent); + return getDAOEvents(affectedParentPerson, affectedParentHost, affectedContent, affectedParentContent, refreshAllContent); } - private Set getDAOEvents(Optional affectedPerson, Host affectedHost, Content affectedContent, boolean triggerFullRefresh) { + private Set getDAOEvents(Optional affectedPerson, Host affectedHost, Content affectedContent, Content affectedParentContent, boolean triggerFullRefresh) { List daoEvents = new ArrayList<>(); if (triggerFullRefresh) { daoEvents.add(new FileSystemContentEvent(null, null, null)); } else if (affectedContent != null) { - Long parentContentId = null; Host parentHost = null; try { - parentContentId = (affectedContent instanceof AbstractContent) - ? ((AbstractContent) affectedContent).getParentId().orElse(null) - : null; - parentHost = (affectedContent instanceof DataSource) ? ((DataSource) affectedContent).getHost() : null; @@ -434,7 +431,7 @@ public class FileSystemDAO extends AbstractDAO { logger.log(Level.WARNING, "An error occurred while fetching content id and host id for content with id of: " + affectedContent.getId(), ex); } - daoEvents.add(new FileSystemContentEvent(affectedContent, parentContentId, parentHost)); + daoEvents.add(new FileSystemContentEvent(affectedContent, affectedParentContent == null ? null : affectedParentContent.getId(), parentHost)); } if (affectedHost != null) { From 8b2d12789f34f7348a139ccae4b404a3d6cc2561 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 7 Dec 2021 11:10:23 -0500 Subject: [PATCH 22/24] revert bundle --- .../autopsy/corecomponents/Bundle.properties-MERGED | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED index 468e6d9430..cde18d3900 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED @@ -72,9 +72,9 @@ DataContentViewerHex.totalPageLabel.text_1=100 DataContentViewerHex.pageLabel2.text=Page # Product Information panel -LBL_Description=
\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
+LBL_Description=
\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
Format_OperatingSystem_Value={0} version {1} running on {2} -LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
+LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
SortChooser.dialogTitle=Choose Sort Criteria ThumbnailViewChildren.progress.cancelling=(Cancelling) # {0} - file name @@ -97,7 +97,7 @@ DataContentViewerHex.goToPageTextField.text= DataContentViewerHex.goToPageLabel.text=Go to Page: DataResultViewerThumbnail.imagesLabel.text=Images: DataResultViewerThumbnail.imagesRangeLabel.text=- -DataResultViewerThumbnail.filePathLabel.text=\ \ \ +DataResultViewerThumbnail.filePathLabel.text=\ AdvancedConfigurationDialog.cancelButton.text=Cancel DataArtifactContentViewer.waitText=Retrieving and preparing data, please wait... DataArtifactContentViewer.errorText=Error retrieving result From e5847192289831c1e9a39bca610509b741d58b9b Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 9 Dec 2021 13:45:47 -0500 Subject: [PATCH 23/24] updates for file type extensions and size --- .../autopsy/mainui/datamodel/MainDAO.java | 3 +- .../autopsy/mainui/datamodel/ViewsDAO.java | 191 +++++++++++++----- .../events/FileTypeExtensionsEvent.java | 29 +-- .../datamodel/events/FileTypeSizeEvent.java | 19 +- .../mainui/nodes/TreeChildFactory.java | 44 ++-- .../mainui/nodes/ViewsTypeFactory.java | 44 +++- 6 files changed, 238 insertions(+), 92 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java index 817a3e212c..148c606b46 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java @@ -68,7 +68,8 @@ public class MainDAO extends AbstractDAO { Case.Events.OS_ACCOUNTS_ADDED.toString(), Case.Events.OS_ACCOUNTS_UPDATED.toString(), Case.Events.OS_ACCOUNTS_DELETED.toString(), - Case.Events.OS_ACCT_INSTANCES_ADDED.toString() + Case.Events.OS_ACCT_INSTANCES_ADDED.toString(), + Case.Events.DATA_SOURCE_ADDED.toString() ); private static final long WATCH_RESOLUTION_MILLIS = 30 * 1000; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index f736313298..8804560afa 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet; import java.beans.PropertyChangeEvent; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -142,8 +143,8 @@ public class ViewsDAO extends AbstractDAO { } FileTypeExtensionsEvent extEvt = (FileTypeExtensionsEvent) eventData; - return key.getFilter().equals(extEvt.getExtensionFilter()) - && (key.getDataSourceId() == null || key.getDataSourceId().equals(extEvt.getDataSourceId())); + return (extEvt.getExtensionFilter() == null || key.getFilter().equals(extEvt.getExtensionFilter())) + && (key.getDataSourceId() == null || extEvt.getDataSourceId() == null || key.getDataSourceId().equals(extEvt.getDataSourceId())); } private boolean isFilesByMimeInvalidating(FileTypeMimeSearchParams key, DAOEvent eventData) { @@ -162,8 +163,8 @@ public class ViewsDAO extends AbstractDAO { } FileTypeSizeEvent sizeEvt = (FileTypeSizeEvent) eventData; - return sizeEvt.getSizeFilter().equals(key.getSizeFilter()) - && (key.getDataSourceId() == null || Objects.equals(key.getDataSourceId(), sizeEvt.getDataSourceId())); + return (sizeEvt.getSizeFilter() == null || sizeEvt.getSizeFilter().equals(key.getSizeFilter())) + && (key.getDataSourceId() == null || sizeEvt.getDataSourceId() == null || Objects.equals(key.getDataSourceId(), sizeEvt.getDataSourceId())); } /** @@ -347,11 +348,13 @@ public class ViewsDAO extends AbstractDAO { for (DAOEvent evt : this.treeCounts.getEnqueued()) { if (evt instanceof FileTypeExtensionsEvent) { FileTypeExtensionsEvent extEvt = (FileTypeExtensionsEvent) evt; - if (dataSourceId == null || Objects.equals(extEvt.getDataSourceId(), dataSourceId)) { - for (FileExtSearchFilter filter : filters) { - if (filter.getFilter().contains(evt)) { - indeterminateFilters.add(filter); - } + if (dataSourceId == null || extEvt.getDataSourceId() == null || Objects.equals(extEvt.getDataSourceId(), dataSourceId)) { + if (extEvt.getExtensionFilter() == null) { + // add all filters if extension filter is null and keep going + indeterminateFilters.addAll(filters); + break; + } else if (filters.contains(extEvt.getExtensionFilter())) { + indeterminateFilters.add(extEvt.getExtensionFilter()); } } } @@ -392,7 +395,7 @@ public class ViewsDAO extends AbstractDAO { FileTypeExtensionsSearchParams.getTypeId(), new FileTypeExtensionsSearchParams(filter, dataSourceId), filter, - filter.getDisplayName(), + filter == null ? "" : filter.getDisplayName(), displayCount); } @@ -412,8 +415,14 @@ public class ViewsDAO extends AbstractDAO { for (DAOEvent evt : this.treeCounts.getEnqueued()) { if (evt instanceof FileTypeSizeEvent) { FileTypeSizeEvent sizeEvt = (FileTypeSizeEvent) evt; - if (dataSourceId == null || Objects.equals(sizeEvt.getDataSourceId(), dataSourceId)) { - indeterminateFilters.add(sizeEvt.getSizeFilter()); + if (dataSourceId == null || sizeEvt.getDataSourceId() == null || Objects.equals(sizeEvt.getDataSourceId(), dataSourceId)) { + if (sizeEvt.getSizeFilter() == null) { + // if null size filter, indicates full refresh and all file sizes need refresh. + indeterminateFilters.addAll(Arrays.asList(FileSizeFilter.values())); + break; + } else { + indeterminateFilters.add(sizeEvt.getSizeFilter()); + } } } } @@ -453,7 +462,7 @@ public class ViewsDAO extends AbstractDAO { FileTypeSizeSearchParams.getTypeId(), new FileTypeSizeSearchParams(filter, dataSourceId), filter, - filter.getDisplayName(), + filter == null ? "" : filter.getDisplayName(), displayCount); } @@ -774,8 +783,21 @@ public class ViewsDAO extends AbstractDAO { @Override Set handleIngestComplete() { - return SubDAOUtils.getIngestCompleteEvents(this.treeCounts, + SubDAOUtils.invalidateKeys(this.searchParamsCache, + (searchParams) -> searchParamsMatchEvent(null, null, null, null, true, searchParams)); + + Set treeEvts = SubDAOUtils.getIngestCompleteEvents(this.treeCounts, (daoEvt, count) -> createTreeItem(daoEvt, count)); + + Set fileViewRefreshEvents = getFileViewRefreshEvents(null); + + List fileViewRefreshTreeEvents = fileViewRefreshEvents.stream() + .map(evt -> new TreeEvent(createTreeItem(evt, TreeDisplayCount.UNSPECIFIED), true)) + .collect(Collectors.toList()); + + return Stream.of(treeEvts, fileViewRefreshEvents, fileViewRefreshTreeEvents) + .flatMap(c -> c.stream()) + .collect(Collectors.toSet()); } @Override @@ -786,53 +808,85 @@ public class ViewsDAO extends AbstractDAO { @Override Set processEvent(PropertyChangeEvent evt) { - AbstractFile af = DAOEventUtils.getFileFromFileEvent(evt); - if (af == null) { - return Collections.emptySet(); - } else if (hideKnownFilesInViewsTree() && TskData.FileKnown.KNOWN.equals(af.getKnown())) { - return Collections.emptySet(); - } + Long dsId = null; + boolean dataSourceAdded = false; + Set evtExtFilters = null; + String evtMimeType = null; + FileSizeFilter evtFileSize = null; - long dsId = af.getDataSourceObjectId(); + if (Case.Events.DATA_SOURCE_ADDED.toString().equals(evt.getPropertyName())) { + dsId = evt.getNewValue() instanceof Long ? (Long) evt.getNewValue() : null; + dataSourceAdded = true; + } else { + AbstractFile af = DAOEventUtils.getFileFromFileEvent(evt); + if (af == null) { + return Collections.emptySet(); + } else if (hideKnownFilesInViewsTree() && TskData.FileKnown.KNOWN.equals(af.getKnown())) { + return Collections.emptySet(); + } - // create an extension mapping if extension present - Set evtExtFilters = (StringUtils.isBlank(af.getNameExtension()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) - ? Collections.emptySet() - : EXTENSION_FILTER_MAP.getOrDefault("." + af.getNameExtension(), Collections.emptySet()); + dsId = af.getDataSourceObjectId(); - // create a mime type mapping if mime type present - String evtMimeType = (StringUtils.isBlank(af.getMIMEType()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) || !getMimeDbFilesTypes().contains(af.getType()) - ? null - : af.getMIMEType(); + // create an extension mapping if extension present + if (StringUtils.isBlank(af.getNameExtension()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) { + evtExtFilters = EXTENSION_FILTER_MAP.getOrDefault("." + af.getNameExtension(), Collections.emptySet()); + } - // create a size mapping if size present in filters - FileSizeFilter evtFileSize = TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType()) - ? null - : Stream.of(FileSizeFilter.values()) + // create a mime type mapping if mime type present + if (StringUtils.isBlank(af.getMIMEType()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType()) || !getMimeDbFilesTypes().contains(af.getType())) { + evtMimeType = af.getMIMEType(); + } + + // create a size mapping if size present in filters + if (TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType())) { + evtFileSize = Stream.of(FileSizeFilter.values()) .filter(filter -> af.getSize() >= filter.getMinBound() && (filter.getMaxBound() == null || af.getSize() < filter.getMaxBound())) .findFirst() .orElse(null); + } - if (evtExtFilters.isEmpty() && evtMimeType == null && evtFileSize == null) { - return Collections.emptySet(); + if (evtExtFilters == null || evtExtFilters.isEmpty() && evtMimeType == null && evtFileSize == null) { + return Collections.emptySet(); + } } - SubDAOUtils.invalidateKeys(this.searchParamsCache, - (Predicate) (searchParams) -> searchParamsMatchEvent(evtExtFilters, evtMimeType, evtFileSize, dsId, searchParams)); + return invalidateAndReturnEvents(evtExtFilters, evtMimeType, evtFileSize, dsId, dataSourceAdded); + } - return getDAOEvents(evtExtFilters, evtMimeType, evtFileSize, dsId); + /** + * Handles invalidating caches and returning events based on digest. + * + * @param evtExtFilters The file extension filters or empty set. + * @param evtMimeType The mime type or null. + * @param evtFileSize The file size filter or null. + * @param dsId The data source id or null. + * @param dataSourceAdded Whether or not this is a data source added event. + * + * @return The set of dao events to be fired. + */ + private Set invalidateAndReturnEvents(Set evtExtFilters, String evtMimeType, + FileSizeFilter evtFileSize, Long dsId, boolean dataSourceAdded) { + + SubDAOUtils.invalidateKeys(this.searchParamsCache, + (Predicate) (searchParams) -> searchParamsMatchEvent(evtExtFilters, evtMimeType, + evtFileSize, dsId, dataSourceAdded, searchParams)); + + return getDAOEvents(evtExtFilters, evtMimeType, evtFileSize, dsId, dataSourceAdded); } private boolean searchParamsMatchEvent(Set evtExtFilters, String evtMimeType, FileSizeFilter evtFileSize, - long dsId, + Long dsId, + boolean dataSourceAdded, Object searchParams) { if (searchParams instanceof FileTypeExtensionsSearchParams) { FileTypeExtensionsSearchParams extParams = (FileTypeExtensionsSearchParams) searchParams; - return evtExtFilters.contains(extParams.getFilter()) - && (extParams.getDataSourceId() == null || Objects.equals(extParams.getDataSourceId(), dsId)); + // if data source added or evtExtFilters contain param filter + return (dataSourceAdded || (evtExtFilters != null && evtExtFilters.contains(extParams.getFilter()))) + // and data source is either null or they are equal data source ids + && (extParams.getDataSourceId() == null || dsId == null || Objects.equals(extParams.getDataSourceId(), dsId)); } else if (searchParams instanceof FileTypeMimeSearchParams) { FileTypeMimeSearchParams mimeParams = (FileTypeMimeSearchParams) searchParams; @@ -841,8 +895,10 @@ public class ViewsDAO extends AbstractDAO { } else if (searchParams instanceof FileTypeSizeSearchParams) { FileTypeSizeSearchParams sizeParams = (FileTypeSizeSearchParams) searchParams; - return Objects.equals(sizeParams.getSizeFilter(), evtFileSize) - && (sizeParams.getDataSourceId() == null || Objects.equals(sizeParams.getDataSourceId(), dsId)); + // if data source added or size filter is equal to param filter + return (dataSourceAdded || Objects.equals(sizeParams.getSizeFilter(), evtFileSize)) + // and data source is either null or they are equal data source ids + && (sizeParams.getDataSourceId() == null || dsId == null || Objects.equals(sizeParams.getDataSourceId(), dsId)); } else { return false; } @@ -852,21 +908,21 @@ public class ViewsDAO extends AbstractDAO { * Clears relevant cache entries from cache based on digest of autopsy * events. * - * @param extFilters The set of affected extension filters. - * @param mimeType The affected mime type or null. - * @param sizeFilter The affected size filter or null. - * @param dsId The file object id. + * @param extFilters The set of affected extension filters. + * @param mimeType The affected mime type or null. + * @param sizeFilter The affected size filter or null. + * @param dsId The file object id. + * @param dataSourceAdded A data source was added. * * @return The list of affected dao events. */ - private Set getDAOEvents(Set extFilters, - String mimeType, - FileSizeFilter sizeFilter, - long dsId) { + private Set getDAOEvents(Set extFilters, String mimeType, FileSizeFilter sizeFilter, Long dsId, boolean dataSourceAdded) { - List daoEvents = extFilters.stream() - .map(extFilter -> new FileTypeExtensionsEvent(extFilter, dsId)) - .collect(Collectors.toList()); + List daoEvents = extFilters == null + ? new ArrayList<>() + : extFilters.stream() + .map(extFilter -> new FileTypeExtensionsEvent(extFilter, dsId)) + .collect(Collectors.toList()); if (mimeType != null) { daoEvents.add(new FileTypeMimeEvent(mimeType, dsId)); @@ -880,11 +936,36 @@ public class ViewsDAO extends AbstractDAO { .map(daoEvt -> new TreeEvent(createTreeItem(daoEvt, TreeDisplayCount.INDETERMINATE), false)) .collect(Collectors.toList()); - return Stream.of(daoEvents, treeEvents) + // data source added events are not necessarily fired before ingest completed/cancelled, so don't handle dataSourceAdded events with delay. + Set forceRefreshEvents = (dataSourceAdded) + ? getFileViewRefreshEvents(dsId) + : Collections.emptySet(); + + List forceRefreshTreeEvents = forceRefreshEvents.stream() + .map(evt -> new TreeEvent(createTreeItem(evt, TreeDisplayCount.UNSPECIFIED), true)) + .collect(Collectors.toList()); + + return Stream.of(daoEvents, treeEvents, forceRefreshEvents, forceRefreshTreeEvents) .flatMap(lst -> lst.stream()) .collect(Collectors.toSet()); } + /** + * Returns events for when a full refresh is required because module content + * events will not necessarily provide events for files (i.e. data source + * added, ingest cancelled/completed). + * + * @param dataSourceId The data source id or null if not applicable. + * + * @return The set of events that apply in this situation. + */ + private Set getFileViewRefreshEvents(Long dataSourceId) { + return ImmutableSet.of( + new FileTypeSizeEvent(null, dataSourceId), + new FileTypeExtensionsEvent(null, dataSourceId) + ); + } + /** * Handles fetching and paging of data for file types by extension. */ diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java index 21a389975a..354cb25010 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeExtensionsEvent.java @@ -22,15 +22,22 @@ import java.util.Objects; import org.sleuthkit.autopsy.mainui.datamodel.FileExtSearchFilter; /** - * An event to signal that files have been added or removed - * with the given extension on the given data source. + * An event to signal that files have been added or removed with the given + * extension on the given data source. */ public class FileTypeExtensionsEvent implements DAOEvent { private final FileExtSearchFilter extensionFilter; - private final long dataSourceId; + private final Long dataSourceId; - public FileTypeExtensionsEvent(FileExtSearchFilter extensionFilter, long dataSourceId) { + /** + * Main constructor. + * + * @param extensionFilter The extension filter. If null, indicates full + * refresh necessary. + * @param dataSourceId The data source id. + */ + public FileTypeExtensionsEvent(FileExtSearchFilter extensionFilter, Long dataSourceId) { this.extensionFilter = extensionFilter; this.dataSourceId = dataSourceId; } @@ -39,15 +46,15 @@ public class FileTypeExtensionsEvent implements DAOEvent { return extensionFilter; } - public long getDataSourceId() { + public Long getDataSourceId() { return dataSourceId; } @Override public int hashCode() { - int hash = 7; - hash = 83 * hash + Objects.hashCode(this.extensionFilter); - hash = 83 * hash + (int) (this.dataSourceId ^ (this.dataSourceId >>> 32)); + int hash = 3; + hash = 89 * hash + Objects.hashCode(this.extensionFilter); + hash = 89 * hash + Objects.hashCode(this.dataSourceId); return hash; } @@ -63,17 +70,15 @@ public class FileTypeExtensionsEvent implements DAOEvent { return false; } final FileTypeExtensionsEvent other = (FileTypeExtensionsEvent) obj; - if (this.dataSourceId != other.dataSourceId) { + if (!Objects.equals(this.extensionFilter, other.extensionFilter)) { return false; } - if (!Objects.equals(this.extensionFilter, other.extensionFilter)) { + if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { return false; } return true; } - - @Override public Type getType() { return Type.RESULT; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeSizeEvent.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeSizeEvent.java index eb0f37f8a4..1fe4256130 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeSizeEvent.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/events/FileTypeSizeEvent.java @@ -22,14 +22,21 @@ import java.util.Objects; import org.sleuthkit.autopsy.mainui.datamodel.FileSizeFilter; /** - * An event to signal that files have been added or removed - * within the given size range on the given data source. + * An event to signal that files have been added or removed within the given + * size range on the given data source. */ public class FileTypeSizeEvent implements DAOEvent { private final FileSizeFilter sizeFilter; private final Long dataSourceId; + /** + * Main constructor. + * + * @param sizeFilter The size filter. If null, indicates full refresh is + * necessary. + * @param dataSourceId The data source id or null. + */ public FileTypeSizeEvent(FileSizeFilter sizeFilter, Long dataSourceId) { this.sizeFilter = sizeFilter; this.dataSourceId = dataSourceId; @@ -45,9 +52,9 @@ public class FileTypeSizeEvent implements DAOEvent { @Override public int hashCode() { - int hash = 7; - hash = 53 * hash + Objects.hashCode(this.sizeFilter); - hash = 53 * hash + Objects.hashCode(this.dataSourceId); + int hash = 5; + hash = 73 * hash + Objects.hashCode(this.sizeFilter); + hash = 73 * hash + Objects.hashCode(this.dataSourceId); return hash; } @@ -72,6 +79,8 @@ public class FileTypeSizeEvent implements DAOEvent { return true; } + + @Override public Type getType() { return Type.RESULT; diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java index 02cbdf1513..0c339674fe 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/TreeChildFactory.java @@ -48,21 +48,7 @@ public abstract class TreeChildFactory extends ChildFactory.Detachable { if (evt.getNewValue() instanceof DAOAggregateEvent) { - DAOAggregateEvent aggEvt = (DAOAggregateEvent) evt.getNewValue(); - for (DAOEvent daoEvt : aggEvt.getEvents()) { - if (daoEvt instanceof TreeEvent) { - TreeEvent treeEvt = (TreeEvent) daoEvt; - TreeItemDTO item = getOrCreateRelevantChild(treeEvt); - if (item != null) { - if (treeEvt.isRefreshRequired()) { - update(); - break; - } else { - updateNodeData(item); - } - } - } - } + handleDAOAggregateEvent((DAOAggregateEvent) evt.getNewValue()); } }; @@ -83,6 +69,30 @@ public abstract class TreeChildFactory extends ChildFactory.Detachable> idMapping = new HashMap<>(); + /** + * Handles processing and updating due to an aggregate event. This method + * can be overridden for custom behavior while handling DAO aggregate + * events. + * + * @param aggEvt The aggregate event. + */ + protected void handleDAOAggregateEvent(DAOAggregateEvent aggEvt) { + for (DAOEvent daoEvt : aggEvt.getEvents()) { + if (daoEvt instanceof TreeEvent) { + TreeEvent treeEvt = (TreeEvent) daoEvt; + TreeItemDTO item = getOrCreateRelevantChild(treeEvt); + if (item != null) { + if (treeEvt.isRefreshRequired()) { + update(); + break; + } else { + updateNodeData(item); + } + } + } + } + } + @Override protected boolean createKeys(List toPopulate) { List> itemsList; @@ -98,9 +108,9 @@ public abstract class TreeChildFactory extends ChildFactory.Detachable(curItemsList); + itemsList = new ArrayList<>(curItemsList); } - + // update existing cached nodes List curResultIds = new ArrayList<>(); for (TreeItemDTO dto : itemsList) { diff --git a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java index e59903225b..373a520088 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/nodes/ViewsTypeFactory.java @@ -37,6 +37,8 @@ import org.sleuthkit.autopsy.mainui.datamodel.FileTypeSizeSearchParams; import org.sleuthkit.autopsy.mainui.datamodel.MainDAO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO; import org.sleuthkit.autopsy.mainui.datamodel.TreeResultsDTO.TreeItemDTO; +import org.sleuthkit.autopsy.mainui.datamodel.events.DAOAggregateEvent; +import org.sleuthkit.autopsy.mainui.datamodel.events.DAOEvent; import org.sleuthkit.autopsy.mainui.datamodel.events.TreeEvent; /** @@ -73,12 +75,32 @@ public class ViewsTypeFactory { return MainDAO.getInstance().getViewsDAO().getFileSizeCounts(this.dataSourceId); } + @Override + protected void handleDAOAggregateEvent(DAOAggregateEvent aggEvt) { + for (DAOEvent evt : aggEvt.getEvents()) { + if (evt instanceof TreeEvent) { + TreeResultsDTO.TreeItemDTO treeItem = super.getTypedTreeItem((TreeEvent) evt, FileTypeSizeSearchParams.class); + // if file type size search params has null filter, trigger full refresh + if (treeItem != null && treeItem.getSearchParams().getSizeFilter() == null) { + super.update(); + return; + } + } + } + + super.handleDAOAggregateEvent(aggEvt); + } + @Override protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, FileTypeSizeSearchParams.class); if (originalTreeItem != null - && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { + // only create child if size filter is present (if null, update should be triggered separately) + && originalTreeItem.getSearchParams().getSizeFilter() != null + && (this.dataSourceId == null + || originalTreeItem.getSearchParams().getDataSourceId() == null + || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { // generate new type so that if it is a subtree event (i.e. keyword hits), the right tree item is created. FileTypeSizeSearchParams searchParam = originalTreeItem.getSearchParams(); @@ -159,7 +181,7 @@ public class ViewsTypeFactory { if (indexOfSlash >= 0) { mimePrefix = mimePrefix.substring(0, indexOfSlash); } - + return new TreeResultsDTO.TreeItemDTO<>( AnalysisResultSearchParam.getTypeId(), new FileTypeMimeSearchParams(mimePrefix, this.dataSourceId), @@ -326,11 +348,29 @@ public class ViewsTypeFactory { return MainDAO.getInstance().getViewsDAO().getFileExtCounts(this.childFilters, this.dataSourceId); } + @Override + protected void handleDAOAggregateEvent(DAOAggregateEvent aggEvt) { + for (DAOEvent evt : aggEvt.getEvents()) { + if (evt instanceof TreeEvent) { + TreeResultsDTO.TreeItemDTO treeItem = super.getTypedTreeItem((TreeEvent) evt, FileTypeExtensionsSearchParams.class); + // if search params has null filter, trigger full refresh + if (treeItem != null && treeItem.getSearchParams().getFilter() == null) { + super.update(); + return; + } + } + } + + super.handleDAOAggregateEvent(aggEvt); + } + @Override protected TreeResultsDTO.TreeItemDTO getOrCreateRelevantChild(TreeEvent treeEvt) { TreeResultsDTO.TreeItemDTO originalTreeItem = super.getTypedTreeItem(treeEvt, FileTypeExtensionsSearchParams.class); if (originalTreeItem != null + // if filter is null, this should trigger a full refresh which should be handled in handleDAOAggregateEvent + && originalTreeItem.getSearchParams().getFilter() != null && this.childFilters.contains(originalTreeItem.getSearchParams().getFilter()) && (this.dataSourceId == null || Objects.equals(this.dataSourceId, originalTreeItem.getSearchParams().getDataSourceId()))) { From 5a397163391532d3d67a613686696e0aa4920008 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 10 Dec 2021 09:45:49 -0500 Subject: [PATCH 24/24] logic fix --- .../org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index 8804560afa..f4e856b82e 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -828,17 +828,17 @@ public class ViewsDAO extends AbstractDAO { dsId = af.getDataSourceObjectId(); // create an extension mapping if extension present - if (StringUtils.isBlank(af.getNameExtension()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) { + if (!StringUtils.isBlank(af.getNameExtension()) && TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) { evtExtFilters = EXTENSION_FILTER_MAP.getOrDefault("." + af.getNameExtension(), Collections.emptySet()); } // create a mime type mapping if mime type present - if (StringUtils.isBlank(af.getMIMEType()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType()) || !getMimeDbFilesTypes().contains(af.getType())) { + if (!StringUtils.isBlank(af.getMIMEType()) && TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType()) && getMimeDbFilesTypes().contains(af.getType())) { evtMimeType = af.getMIMEType(); } // create a size mapping if size present in filters - if (TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType())) { + if (!TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType())) { evtFileSize = Stream.of(FileSizeFilter.values()) .filter(filter -> af.getSize() >= filter.getMinBound() && (filter.getMaxBound() == null || af.getSize() < filter.getMaxBound())) .findFirst()