codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-09-06 02:24:30 +00:00
Code Issues Releases Wiki Activity

WIP to use sqlite ATTACH DATABASE to implement tag filters

Browse Source
This commit is contained in:
jmillman
2015-08-05 16:57:04 -04:00
parent f595d3fe12
commit 7744cf7fd2
12 changed files with 275 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
Core/src/org/sleuthkit/autopsy/timeline/events/FilteredEventsModel.java
View File

@@ -27,6 +27,7 @@ import java.util.logging.Level;
import javafx.beans.Observable;
import javafx.beans.property.ReadOnlyObjectProperty;
import javafx.beans.property.ReadOnlyObjectWrapper;
import javafx.collections.ListChangeListener;
import javafx.collections.MapChangeListener;
import javax.annotation.concurrent.GuardedBy;
import org.joda.time.DateTimeZone;
@@ -48,6 +49,8 @@ import org.sleuthkit.autopsy.timeline.filters.HashHitsFilter;
import org.sleuthkit.autopsy.timeline.filters.HashSetFilter;
import org.sleuthkit.autopsy.timeline.filters.HideKnownFilter;
import org.sleuthkit.autopsy.timeline.filters.RootFilter;
import org.sleuthkit.autopsy.timeline.filters.TagNameFilter;
import org.sleuthkit.autopsy.timeline.filters.TagsFilter;
import org.sleuthkit.autopsy.timeline.filters.TextFilter;
import org.sleuthkit.autopsy.timeline.filters.TypeFilter;
import org.sleuthkit.autopsy.timeline.zooming.DescriptionLOD;
@@ -55,6 +58,7 @@ import org.sleuthkit.autopsy.timeline.zooming.EventTypeZoomLevel;
import org.sleuthkit.autopsy.timeline.zooming.ZoomParams;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TagName;
import org.sleuthkit.datamodel.TskCoreException;
/**
@@ -131,7 +135,14 @@ public final class FilteredEventsModel {
hashSourceFilter.setSelected(Boolean.TRUE);
hashHitsFilter.addHashSetFilter(hashSourceFilter);
});
return new RootFilter(new HideKnownFilter(), hashHitsFilter, new TextFilter(), new TypeFilter(RootEventType.getInstance()), dataSourcesFilter);
TagsFilter tagsFilter = new TagsFilter();
repo.getTagNames().stream().forEach(t -> {
TagNameFilter tagNameFilter = new TagNameFilter(t);
tagNameFilter.setSelected(Boolean.TRUE);
tagsFilter.addTagFilter(tagNameFilter);
});
return new RootFilter(new HideKnownFilter(), tagsFilter, hashHitsFilter, new TextFilter(), new TypeFilter(RootEventType.getInstance()), dataSourcesFilter);
}
public FilteredEventsModel(EventsRepository repo, ReadOnlyObjectProperty<ZoomParams> currentStateProperty) {
@@ -149,6 +160,19 @@ public final class FilteredEventsModel {
rootFilter.getHashHitsFilter().addHashSetFilter(hashSetFilter);
requestedFilter.set(rootFilter.copyOf());
});
repo.getTagNames().addListener((ListChangeListener.Change<? extends TagName> c) -> {
RootFilter rootFilter = filter().get();
TagsFilter tagsFilter = rootFilter.getTagsFilter();
while (c.next()) {
c.getRemoved().forEach(tagsFilter::removeFilterForTag);
c.getAddedSubList().forEach((TagName t) -> {
TagNameFilter tagFilter = new TagNameFilter(t);
tagsFilter.addTagFilter(tagFilter);
});
}
requestedFilter.set(rootFilter.copyOf());
});
requestedFilter.set(getDefaultFilter());
requestedZoomParamters.addListener((Observable observable) -> {

20
Core/src/org/sleuthkit/autopsy/timeline/events/db/EventDB.java
View File

@@ -248,7 +248,7 @@ public class EventDB {
*/
Map<EventType, Long> countEventsByType(ZoomParams params) {
if (params.getTimeRange() != null) {
return countEvents(params.getTimeRange().getStartMillis() / 1000,
return countEventsByType(params.getTimeRange().getStartMillis() / 1000,
params.getTimeRange().getEndMillis() / 1000,
params.getFilter(), params.getTypeZoomLevel());
} else {
@@ -791,6 +791,17 @@ public class EventDB {
DBLock.lock();
//this should match Sleuthkit db setupt
try (Statement statement = con.createStatement()) {
String toString = Paths.get(dbPath).getParent().resolve("autopsy.db").toString();
System.out.println(toString);
ResultSet rs = statement.executeQuery("PRAGMA database_list");
boolean found = false;
while (rs.next() && !found) {
found |= "autopsy".equalsIgnoreCase(rs.getString("name"));
}
if (!found) {
statement.execute("ATTACH DATABASE 'file:" + toString + "?mode=ro' AS autopsy");
}
//reduce i/o operations, we have no OS crash recovery anyway
statement.execute("PRAGMA synchronous = OFF;"); // NON-NLS
//we don't use this feature, so turn it off for minimal speed up on queries
@@ -846,7 +857,7 @@ public class EventDB {
* @return a map organizing the counts in a hierarchy from date > eventtype>
* count
*/
private Map<EventType, Long> countEvents(Long startTime, Long endTime, RootFilter filter, EventTypeZoomLevel zoomLevel) {
private Map<EventType, Long> countEventsByType(Long startTime, Long endTime, RootFilter filter, EventTypeZoomLevel zoomLevel) {
if (Objects.equals(startTime, endTime)) {
endTime++;
}
@@ -931,7 +942,7 @@ public class EventDB {
DBLock.lock();
String query = "select strftime('" + strfTimeFormat + "',time , 'unixepoch'" + (TimeLineController.getTimeZone().get().equals(TimeZone.getDefault()) ? ", 'localtime'" : "") + ") as interval,"
+ " group_concat(events.event_id) as event_ids, Min(time), Max(time), " + descriptionColumn + ", " + useSubTypeHelper(useSubTypes)
+ " from events" + useHashHitTablesHelper(filter) + " where " + "time >= " + start + " and time < " + end + " and " + SQLHelper.getSQLWhere(filter) // NON-NLS
+ " from events" + useHashHitTablesHelper(filter) + useTagTablesHelper(filter)+ " where " + "time >= " + start + " and time < " + end + " and " + SQLHelper.getSQLWhere(filter) // NON-NLS
+ " group by interval, " + useSubTypeHelper(useSubTypes) + " , " + descriptionColumn // NON-NLS
+ " order by Min(time)"; // NON-NLS
// scoop up requested events in groups organized by interval, type, and desription
@@ -1022,6 +1033,9 @@ public class EventDB {
private String useHashHitTablesHelper(RootFilter filter) {
return SQLHelper.hasActiveHashFilter(filter) ? ", hash_set_hits" : "";
}
private String useTagTablesHelper(RootFilter filter) {
return SQLHelper.hasActiveTagFilter(filter) ? ", content_tags, blackboard_artifact_tags " : "";
}
private static String useSubTypeHelper(final boolean useSubTypes) {
return useSubTypes ? "sub_type" : "base_type";

32
Core/src/org/sleuthkit/autopsy/timeline/events/db/EventsRepository.java
View File

@@ -34,6 +34,7 @@ import java.util.logging.Level;
import java.util.stream.Collectors;
import javafx.beans.property.ReadOnlyObjectProperty;
import javafx.collections.FXCollections;
import javafx.collections.ObservableList;
import javafx.collections.ObservableMap;
import javax.annotation.concurrent.GuardedBy;
import javax.swing.JOptionPane;
@@ -57,6 +58,7 @@ import org.sleuthkit.autopsy.timeline.zooming.ZoomParams;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TagName;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
@@ -95,12 +97,17 @@ public class EventsRepository {
private final ObservableMap<Long, String> datasourcesMap = FXCollections.observableHashMap();
private final ObservableMap<Long, String> hashSetMap = FXCollections.observableHashMap();
private final ObservableList<TagName> tagNames = FXCollections.observableArrayList();
private final Case autoCase;
public Case getAutoCase() {
return autoCase;
}
public ObservableList<TagName> getTagNames() {
return tagNames;
}
synchronized public ObservableMap<Long, String> getDatasourcesMap() {
return datasourcesMap;
}
@@ -125,7 +132,7 @@ public class EventsRepository {
this.autoCase = autoCase;
//TODO: we should check that case is open, or get passed a case object/directory -jm
this.eventDB = EventDB.getEventDB(autoCase);
populateFilterMaps(autoCase.getSleuthkitCase());
populateFilterData(autoCase.getSleuthkitCase());
idToEventCache = CacheBuilder.newBuilder()
.maximumSize(5000L)
.expireAfterAccess(10, TimeUnit.MINUTES)
@@ -289,8 +296,8 @@ public class EventsRepository {
String rootFolder = StringUtils.substringBetween(parentPath, "/", "/");
String shortDesc = datasourceName + "/" + StringUtils.defaultIfBlank(rootFolder, "");
String medD = datasourceName + parentPath;
final TskData.FileKnown known = f.getKnown();
Set<String> hashSets = f.getHashSetNames() ;
final TskData.FileKnown known = f.getKnown();
Set<String> hashSets = f.getHashSetNames();
boolean tagged = !tagsManager.getContentTagsByContent(f).isEmpty();
//insert it into the db if time is > 0 => time is legitimate (drops logical files)
@@ -336,7 +343,7 @@ public class EventsRepository {
eventDB.commitTransaction(trans, true);
}
populateFilterMaps(skCase);
populateFilterData(skCase);
invalidateCaches();
return null;
@@ -400,7 +407,7 @@ public class EventsRepository {
long datasourceID = skCase.getContentById(bbart.getObjectID()).getDataSource().getId();
AbstractFile f = skCase.getAbstractFileById(bbart.getObjectID());
Set<String> hashSets = f.getHashSetNames();
Set<String> hashSets = f.getHashSetNames();
boolean tagged = tagsManager.getBlackboardArtifactTagsByArtifact(bbart).isEmpty() == false;
eventDB.insertEvent(eventDescription.getTime(), type, datasourceID, bbart.getObjectID(), bbart.getArtifactID(), eventDescription.getFullDescription(), eventDescription.getMedDescription(), eventDescription.getShortDescription(), null, hashSets, tagged, trans);
@@ -425,7 +432,7 @@ public class EventsRepository {
*
* @param skCase
*/
synchronized private void populateFilterMaps(SleuthkitCase skCase) {
synchronized private void populateFilterData(SleuthkitCase skCase) {
for (Map.Entry<Long, String> hashSet : eventDB.getHashSetNames().entrySet()) {
hashSetMap.putIfAbsent(hashSet.getKey(), hashSet.getValue());
@@ -438,13 +445,24 @@ public class EventsRepository {
LOGGER.log(Level.SEVERE, "Failed to get datasource by ID.", ex);
}
}
try {
tagNames.setAll(skCase.getTagNamesInUse());
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Failed to get tag names in use.", ex);
}
}
synchronized public Set<Long> markEventsTagged(long objID, Long artifactID, boolean tagged) {
synchronized public Set<Long> markEventsTagged(long objID, Long artifactID, boolean tagged) {
Set<Long> updatedEventIDs = eventDB.markEventsTagged(objID, artifactID, tagged);
if (!updatedEventIDs.isEmpty()) {
aggregateEventsCache.invalidateAll();
idToEventCache.invalidateAll(updatedEventIDs);
try {
tagNames.setAll(autoCase.getSleuthkitCase().getTagNamesInUse());
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Failed to get tag names in use.", ex);
}
}
return updatedEventIDs;
}

30
Core/src/org/sleuthkit/autopsy/timeline/events/db/SQLHelper.java
View File

@@ -19,6 +19,8 @@ import org.sleuthkit.autopsy.timeline.filters.HashSetFilter;
import org.sleuthkit.autopsy.timeline.filters.HideKnownFilter;
import org.sleuthkit.autopsy.timeline.filters.IntersectionFilter;
import org.sleuthkit.autopsy.timeline.filters.RootFilter;
import org.sleuthkit.autopsy.timeline.filters.TagNameFilter;
import org.sleuthkit.autopsy.timeline.filters.TagsFilter;
import org.sleuthkit.autopsy.timeline.filters.TextFilter;
import org.sleuthkit.autopsy.timeline.filters.TypeFilter;
import org.sleuthkit.autopsy.timeline.filters.UnionFilter;
@@ -46,6 +48,11 @@ public class SQLHelper {
return hashHitFilter.isSelected() && false == hashHitFilter.isDisabled();
}
static boolean hasActiveTagFilter(RootFilter filter) {
TagsFilter tagsFilter = filter.getTagsFilter();
return tagsFilter.isSelected() && false == tagsFilter.isDisabled();
}
private SQLHelper() {
}
@@ -61,6 +68,8 @@ public class SQLHelper {
String result = "";
if (filter == null) {
return "1";
} else if (filter instanceof TagsFilter) {
result = getSQLWhere((TagsFilter) filter);
} else if (filter instanceof HashHitsFilter) {
result = getSQLWhere((HashHitsFilter) filter);
} else if (filter instanceof DataSourceFilter) {
@@ -89,7 +98,22 @@ public class SQLHelper {
static String getSQLWhere(HideKnownFilter filter) {
if (filter.isSelected()) {
return "(known_state IS NOT '" + TskData.FileKnown.KNOWN.getFileKnownValue() + "')"; // NON-NLS
return "(known_state IS NOT '" + TskData.FileKnown.KNOWN.getFileKnownValue() + "')"; // NON-NLS
} else {
return "1";
}
}
static String getSQLWhere(TagsFilter filter) {
if (filter.isSelected()
&& (false == filter.isDisabled())
&& (filter.getSubFilters().isEmpty() == false)) {
String collect = filter.getSubFilters().stream()
.filter((TagNameFilter t) -> t.isSelected() && !t.isDisabled())
.map((TagNameFilter t) -> String.valueOf(t.getTagName().getId()))
.collect(Collectors.joining(", ", "(", ")"));
return "((blackboard_artifact_tags.artifact_id == events.artifact_id AND blackboard_artifact_tags.tag_name_id IN " + collect + ") OR ( content_tags.obj_id == events.file_id AND content_tags.tag_name_id IN " + collect + "))";
} else {
return "1";
}
@@ -99,10 +123,10 @@ public class SQLHelper {
if (filter.isSelected()
&& (false == filter.isDisabled())
&& (filter.getSubFilters().isEmpty() == false)) {
return "(hash_set_hits.hash_set_id in " + filter.getSubFilters().stream()
return "(hash_set_hits.hash_set_id IN " + filter.getSubFilters().stream()
.filter((HashSetFilter t) -> t.isSelected() && !t.isDisabled())
.map((HashSetFilter t) -> String.valueOf(t.getHashSetID()))
.collect(Collectors.joining(", ", "(", ")")) + " and hash_set_hits.event_id == events.event_id)";
.collect(Collectors.joining(", ", "(", ")")) + " AND hash_set_hits.event_id == events.event_id)";
} else {
return "1";
}

2
Core/src/org/sleuthkit/autopsy/timeline/filters/CompoundFilter.java
View File

@@ -78,7 +78,7 @@ public abstract class CompoundFilter<SubFilterType extends Filter> extends Abstr
}
}
static <SubFilterType extends Filter> boolean hashEqualSubFilters(final CompoundFilter<SubFilterType> oneFilter, final CompoundFilter<SubFilterType> otherFilter) {
static <SubFilterType extends Filter> boolean areSubFiltersEqual(final CompoundFilter<SubFilterType> oneFilter, final CompoundFilter<SubFilterType> otherFilter) {
if (oneFilter.getSubFilters().size() != otherFilter.getSubFilters().size()) {
return false;
}

2
Core/src/org/sleuthkit/autopsy/timeline/filters/DataSourcesFilter.java
View File

@@ -90,7 +90,7 @@ public class DataSourcesFilter extends UnionFilter<DataSourceFilter> {
return false;
}
return hashEqualSubFilters(this, other);
return areSubFiltersEqual(this, other);
}

2
Core/src/org/sleuthkit/autopsy/timeline/filters/HashHitsFilter.java
View File

@@ -68,7 +68,7 @@ public class HashHitsFilter extends UnionFilter<HashSetFilter> {
return false;
}
return hashEqualSubFilters(this, other);
return areSubFiltersEqual(this, other);
}
public void addHashSetFilter(HashSetFilter hashSetFilter) {

3
Core/src/org/sleuthkit/autopsy/timeline/filters/HashSetFilter.java
View File

@@ -21,7 +21,7 @@ package org.sleuthkit.autopsy.timeline.filters;
import java.util.Objects;
/**
* Filter for an individual datasource
* Filter for an individual hash set
*/
public class HashSetFilter extends AbstractFilter {
@@ -84,5 +84,4 @@ public class HashSetFilter extends AbstractFilter {
}
return isSelected() == other.isSelected();
}
}

14
Core/src/org/sleuthkit/autopsy/timeline/filters/RootFilter.java
View File

@@ -27,6 +27,7 @@ import javafx.collections.FXCollections;
public class RootFilter extends IntersectionFilter<Filter> {
private final HideKnownFilter knwonFilter;
private final TagsFilter tagsFilter;
private final HashHitsFilter hashFilter;
private final TextFilter textFilter;
private final TypeFilter typeFilter;
@@ -36,13 +37,18 @@ public class RootFilter extends IntersectionFilter<Filter> {
return dataSourcesFilter;
}
public TagsFilter getTagsFilter() {
return tagsFilter;
}
public HashHitsFilter getHashHitsFilter() {
return hashFilter;
}
public RootFilter(HideKnownFilter knownFilter, HashHitsFilter hashFilter, TextFilter textFilter, TypeFilter typeFilter, DataSourcesFilter dataSourceFilter) {
super(FXCollections.observableArrayList(knownFilter, hashFilter, textFilter, dataSourceFilter, typeFilter));
public RootFilter(HideKnownFilter knownFilter, TagsFilter tagsFilter, HashHitsFilter hashFilter, TextFilter textFilter, TypeFilter typeFilter, DataSourcesFilter dataSourceFilter) {
super(FXCollections.observableArrayList(knownFilter, tagsFilter, hashFilter, textFilter, dataSourceFilter, typeFilter));
this.knwonFilter = knownFilter;
this.tagsFilter = tagsFilter;
this.hashFilter = hashFilter;
this.textFilter = textFilter;
this.typeFilter = typeFilter;
@@ -51,7 +57,7 @@ public class RootFilter extends IntersectionFilter<Filter> {
@Override
public RootFilter copyOf() {
RootFilter filter = new RootFilter(knwonFilter.copyOf(), hashFilter.copyOf(), textFilter.copyOf(), typeFilter.copyOf(), dataSourcesFilter.copyOf());
RootFilter filter = new RootFilter(knwonFilter.copyOf(), tagsFilter.copyOf(), hashFilter.copyOf(), textFilter.copyOf(), typeFilter.copyOf(), dataSourcesFilter.copyOf());
filter.setSelected(isSelected());
filter.setDisabled(isDisabled());
return filter;
@@ -71,6 +77,6 @@ public class RootFilter extends IntersectionFilter<Filter> {
if (getClass() != obj.getClass()) {
return false;
}
return hashEqualSubFilters(this, (CompoundFilter<Filter>) obj);
return areSubFiltersEqual(this, (CompoundFilter<Filter>) obj);
}
}

79
Core/src/org/sleuthkit/autopsy/timeline/filters/TagNameFilter.java Normal file
View File

@@ -0,0 +1,79 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.timeline.filters;
import java.util.Objects;
import org.sleuthkit.datamodel.TagName;
/**
* Filter for an individual TagName
*/
public class TagNameFilter extends AbstractFilter {
private final TagName tagName;
public TagNameFilter(TagName tagName) {
this.tagName = tagName;
}
public TagName getTagName() {
return tagName;
}
@Override
synchronized public TagNameFilter copyOf() {
TagNameFilter filterCopy = new TagNameFilter(getTagName());
filterCopy.setSelected(isSelected());
filterCopy.setDisabled(isDisabled());
return filterCopy;
}
@Override
public String getDisplayName() {
return tagName.getDisplayName();
}
@Override
public String getHTMLReportString() {
return getDisplayName() + getStringCheckBox();
}
@Override
public int hashCode() {
int hash = 3;
hash = 53 * hash + Objects.hashCode(this.tagName);
return hash;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final TagNameFilter other = (TagNameFilter) obj;
if (!Objects.equals(this.tagName, other.tagName)) {
return false;
}
return isSelected() == other.isSelected();
}
}

87
Core/src/org/sleuthkit/autopsy/timeline/filters/TagsFilter.java Normal file
View File

@@ -0,0 +1,87 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package org.sleuthkit.autopsy.timeline.filters;
import java.util.stream.Collectors;
import javafx.beans.binding.Bindings;
import org.openide.util.NbBundle;
import org.sleuthkit.datamodel.TagName;
/**
*
*/
public class TagsFilter extends UnionFilter<TagNameFilter> {
@Override
@NbBundle.Messages("tagsFilter.displayName.text=Only Events Tagged")
public String getDisplayName() {
return Bundle.tagsFilter_displayName_text();
}
public TagsFilter() {
getDisabledProperty().bind(Bindings.size(getSubFilters()).lessThan(1));
setSelected(false);
}
@Override
public TagsFilter copyOf() {
TagsFilter filterCopy = new TagsFilter();
filterCopy.setSelected(isSelected());
//add a copy of each subfilter
this.getSubFilters().forEach((TagNameFilter t) -> {
filterCopy.addTagFilter(t.copyOf());
});
return filterCopy;
}
@Override
public String getHTMLReportString() {
//move this logic into SaveSnapshot
String string = getDisplayName() + getStringCheckBox();
if (getSubFilters().isEmpty() == false) {
string = string + " : " + getSubFilters().stream()
.filter(Filter::isSelected)
.map(Filter::getHTMLReportString)
.collect(Collectors.joining("</li><li>", "<ul><li>", "</li></ul>")); // NON-NLS
}
return string;
}
@Override
public int hashCode() {
return 7;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final TagsFilter other = (TagsFilter) obj;
if (isSelected() != other.isSelected()) {
return false;
}
return areSubFiltersEqual(this, other);
}
public void addTagFilter(TagNameFilter tagFilter) {
if (getSubFilters().stream().map(TagNameFilter.class::cast)
.map(TagNameFilter::getTagName)
.filter(t -> t == tagFilter.getTagName())
.findAny().isPresent() == false) {
getSubFilters().add(tagFilter);
}
}
public void removeFilterForTag(TagName tagName) {
getSubFilters().removeIf((TagNameFilter t) -> t.getTagName().equals(tagName));
}
}

2
Core/src/org/sleuthkit/autopsy/timeline/filters/TypeFilter.java
View File

@@ -133,7 +133,7 @@ public class TypeFilter extends UnionFilter<TypeFilter> {
if (this.eventType != other.eventType) {
return false;
}
return hashEqualSubFilters(this, other);
return areSubFiltersEqual(this, other);
}
@Override