codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-09-06 02:24:30 +00:00
Code Issues Releases Wiki Activity

WIP to optimise tag name retrieval

Browse Source
This commit is contained in:
jmillman
2015-08-25 17:33:15 -04:00
parent e708a43a93
commit 7ff8149c1e
6 changed files with 243 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java
View File

@@ -297,7 +297,6 @@ public class TimeLineController {
//TODO: verify this locking is correct? -jm
synchronized (eventsRepository) {
eventsRepository.rebuildRepository(() -> {
synchronized (eventsRepository) {
eventsRepository.recordLastObjID(lastObjId);
eventsRepository.recordLastArtifactID(lastArtfID);
@@ -309,7 +308,7 @@ public class TimeLineController {
needsHistogramRebuild.set(false);
showWindow();
}
Platform.runLater(() -> {
//TODO: should this be an event?
newEventsFlag.set(false);
@@ -325,6 +324,25 @@ public class TimeLineController {
return true;
}
boolean rebuildTagsTable() {
LOGGER.log(Level.INFO, "starting to rebuild tags table"); // NON-NLS
SwingUtilities.invokeLater(() -> {
if (isWindowOpen()) {
mainFrame.close();
}
});
synchronized (eventsRepository) {
eventsRepository.rebuildTags(() -> {
showWindow();
Platform.runLater(() -> {
showFullRange();
});
});
}
return true;
}
public void showFullRange() {
synchronized (filteredEvents) {
pushTimeRange(filteredEvents.getSpanningInterval());
@@ -389,14 +407,12 @@ public class TimeLineController {
}
}
}
/*
* if the repo was not rebuilt show the UI. If the repo was rebuild
* it will be displayed as part of that process
* if the repo was not rebuilt at minimum rebuild the tags which may
* have been updated without or knowing it.
*/
if (repoRebuilt == false) {
showWindow();
showFullRange();
rebuildTagsTable();
}
} catch (TskCoreException ex) {

5
Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java
View File

@@ -242,6 +242,10 @@ public final class FilteredEventsModel {
return repo.getEventsById(eventIDs);
}
public Map<String, Long> getTagCountsByTagName(Set<Long> eventIDsWithTags) {
return repo.getTagCountsByTagName(eventIDsWithTags);
}
public Set<Long> getEventIDs(Interval timeRange, Filter filter) {
final Interval overlap;
final RootFilter intersect;
@@ -398,4 +402,5 @@ public final class FilteredEventsModel {
public void refresh() {
eventbus.post(new RefreshRequestedEvent());
}
}

103
Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java
View File

@@ -155,6 +155,7 @@ public class EventDB {
private PreparedStatement dropEventsTableStmt;
private PreparedStatement dropHashSetHitsTableStmt;
private PreparedStatement dropHashSetsTableStmt;
private PreparedStatement dropTagsTableStmt;
private PreparedStatement dropDBInfoTableStmt;
private PreparedStatement selectEventIDsFromOBjectAndArtifactStmt;
@@ -208,11 +209,11 @@ public class EventDB {
return new EventTransaction();
}
void commitTransaction(EventTransaction tr, Boolean notify) {
void commitTransaction(EventTransaction tr) {
if (tr.isClosed()) {
throw new IllegalArgumentException("can't close already closed transaction"); // NON-NLS
}
tr.commit(notify);
tr.commit();
}
/**
@@ -252,6 +253,25 @@ public class EventDB {
}
}
Map<String, Long> getTagCountsByTagName(Set<Long> eventIDsWithTags) {
HashMap<String, Long> counts = new HashMap<>();
try (Statement createStatement = con.createStatement();
ResultSet rs = createStatement.executeQuery("SELECT tag_name_displayName, COUNT(DISTINCT tag_id) AS count FROM tags"
+ " WHERE event_id IN (" + StringUtils.join(eventIDsWithTags, ", ") + ")"
+ " GROUP BY tag_name_id"
+ " ORDER BY tag_name_displayName");) {
while (rs.next()) {
counts.put(rs.getString("tag_name_displayName"), rs.getLong("count"));
}
} catch (SQLException ex) {
Exceptions.printStackTrace(ex);
} finally {
}
return counts;
}
/**
* drop the tables from this database and recreate them in order to start
* over.
@@ -262,10 +282,23 @@ public class EventDB {
dropEventsTableStmt.executeUpdate();
dropHashSetHitsTableStmt.executeUpdate();
dropHashSetsTableStmt.executeUpdate();
dropTagsTableStmt.executeUpdate();
dropDBInfoTableStmt.executeUpdate();
initializeDB();;
initializeDB();
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "could not drop old tables table", ex); // NON-NLS
LOGGER.log(Level.SEVERE, "could not drop old tables", ex); // NON-NLS
} finally {
DBLock.unlock();
}
}
void reInitializeTags() {
DBLock.lock();
try {
dropTagsTableStmt.executeUpdate();
initializeTagsTable();
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "could not drop old tags table", ex); // NON-NLS
} finally {
DBLock.unlock();
}
@@ -535,16 +568,8 @@ public class EventDB {
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "problem creating hash_set_hits table", ex);
}
try (Statement stmt = con.createStatement()) {
String sql = "CREATE TABLE IF NOT EXISTS tags "
+ "(tag_id INTEGER NOT NULL,"
+ " tag_name_id INTEGER NOT NULL, "
+ " event_id INTEGER REFERENCES events(event_id) NOT NULL, "
+ " PRIMARY KEY (event_id, tag_id))";
stmt.execute(sql);
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "problem creating hash_set_hits table", ex);
}
initializeTagsTable();
createIndex("events", Arrays.asList("file_id"));
createIndex("events", Arrays.asList("artifact_id"));
@@ -567,12 +592,13 @@ public class EventDB {
insertHashSetStmt = prepareStatement("INSERT OR IGNORE INTO hash_sets (hash_set_name) values (?)");
selectHashSetStmt = prepareStatement("SELECT hash_set_id FROM hash_sets WHERE hash_set_name = ?");
insertHashHitStmt = prepareStatement("INSERT OR IGNORE INTO hash_set_hits (hash_set_id, event_id) values (?,?)");
insertTagStmt = prepareStatement("INSERT OR IGNORE INTO tags (tag_id, tag_name_id, event_id) values (?,?,?)");
insertTagStmt = prepareStatement("INSERT OR IGNORE INTO tags (tag_id, tag_name_id,tag_name_displayName, event_id) values (?,?,?,?)");
deleteTagStmt = prepareStatement("DELETE FROM tags WHERE tag_id = ?");
countAllEventsStmt = prepareStatement("SELECT count(*) AS count FROM events");
dropEventsTableStmt = prepareStatement("DROP TABLE IF EXISTS events");
dropHashSetHitsTableStmt = prepareStatement("DROP TABLE IF EXISTS hash_set_hits");
dropHashSetsTableStmt = prepareStatement("DROP TABLE IF EXISTS hash_sets");
dropTagsTableStmt = prepareStatement("DROP TABLE IF EXISTS tags");
dropDBInfoTableStmt = prepareStatement("DROP TABLE IF EXISTS db_ino");
selectEventIDsFromOBjectAndArtifactStmt = prepareStatement("SELECT event_id FROM events WHERE file_id == ? AND artifact_id IS ?");
} catch (SQLException sQLException) {
@@ -583,6 +609,20 @@ public class EventDB {
}
}
private void initializeTagsTable() {
try (Statement stmt = con.createStatement()) {
String sql = "CREATE TABLE IF NOT EXISTS tags "
+ "(tag_id INTEGER NOT NULL,"
+ " tag_name_id INTEGER NOT NULL, "
+ " tag_name_displayName TEXT NOT NULL, "
+ " event_id INTEGER REFERENCES events(event_id) NOT NULL, "
+ " PRIMARY KEY (event_id, tag_name_id))";
stmt.execute(sql);
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "problem creating hash_set_hits table", ex);
}
}
/**
*
* @param tableName the value of tableName
@@ -638,7 +678,7 @@ public class EventDB {
EventTransaction transaction = beginTransaction();
insertEvent(time, type, datasourceID, objID, artifactID, fullDescription, medDescription, shortDescription, known, hashSets, tags, transaction);
commitTransaction(transaction, true);
commitTransaction(transaction);
}
/**
@@ -714,11 +754,8 @@ public class EventDB {
}
}
for (Tag tag : tags) {
//"INSERT OR IGNORE INTO tags (tag_id, tag_name_id, event_id) values (?,?,?)");
insertTagStmt.setLong(1, tag.getId());
insertTagStmt.setLong(2, tag.getName().getId());
insertTagStmt.setLong(3, eventID);
insertTagStmt.executeUpdate();
//could this be one insert? is there a performance win?
insertTag(tag, eventID);
}
break;
}
@@ -736,13 +773,7 @@ public class EventDB {
try {
Set<Long> eventIDs = markEventsTagged(objectID, artifactID, true);
for (Long eventID : eventIDs) {
//could this be one insert? is there a performance win?
//"INSERT OR IGNORE INTO tags (tag_id, tag_name_id, event_id) values (?,?,?)"
insertTagStmt.clearParameters();
insertTagStmt.setLong(1, tag.getId());
insertTagStmt.setLong(2, tag.getName().getId());
insertTagStmt.setLong(3, eventID);
insertTagStmt.executeUpdate();
insertTag(tag, eventID);
}
return eventIDs;
} catch (SQLException ex) {
@@ -753,6 +784,17 @@ public class EventDB {
return Collections.emptySet();
}
private void insertTag(Tag tag, Long eventID) throws SQLException {
//"INSERT OR IGNORE INTO tags (tag_id, tag_name_id,tag_name_displayName, event_id) values (?,?,?,?)"
insertTagStmt.clearParameters();
insertTagStmt.setLong(1, tag.getId());
insertTagStmt.setLong(2, tag.getName().getId());
insertTagStmt.setString(3, tag.getName().getDisplayName());
insertTagStmt.setLong(4, eventID);
insertTagStmt.executeUpdate();
}
Set<Long> deleteTag(long objectID, Long artifactID, Tag tag, boolean stillTagged) {
DBLock.lock();
try {
@@ -1177,16 +1219,13 @@ public class EventDB {
}
}
private void commit(Boolean notify) {
private void commit() {
if (!closed) {
try {
con.commit();
// make sure we close before we update, bc they'll need locks
close();
if (notify) {
// fireNewEvents(newEvents);
}
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "Error commiting events.db.", ex); // NON-NLS
rollback();

125
Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java
View File

@@ -22,7 +22,6 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -239,10 +238,121 @@ public class EventsRepository {
dbPopulationWorker.execute();
}
synchronized public void rebuildTags(Runnable r) {
if (dbPopulationWorker != null) {
dbPopulationWorker.cancel(true);
}
dbPopulationWorker = new RebuildTagsWorker(r);
dbPopulationWorker.execute();
}
public boolean hasDataSourceInfo() {
return eventDB.hasNewColumns();
}
public Map<String, Long> getTagCountsByTagName(Set<Long> eventIDsWithTags) {
return eventDB.getTagCountsByTagName(eventIDsWithTags);
}
private class RebuildTagsWorker extends SwingWorker<Void, ProgressWindow.ProgressUpdate> {
private final ProgressWindow progressDialog;
//TODO: can we avoid this with a state listener? does it amount to the same thing?
//post population operation to execute
private final Runnable postPopulationOperation;
private final SleuthkitCase skCase;
private final TagsManager tagsManager;
public RebuildTagsWorker(Runnable postPopulationOperation) {
progressDialog = new ProgressWindow(null, true, this);
progressDialog.setVisible(true);
skCase = autoCase.getSleuthkitCase();
tagsManager = autoCase.getServices().getTagsManager();
this.postPopulationOperation = postPopulationOperation;
}
@Override
protected Void doInBackground() throws Exception {
EventDB.EventTransaction trans = eventDB.beginTransaction();
LOGGER.log(Level.INFO, "dropping old tags"); // NON-NLS
eventDB.reInitializeTags();
LOGGER.log(Level.INFO, "updating content tags"); // NON-NLS
List<ContentTag> contentTags = tagsManager.getAllContentTags();
int size = contentTags.size();
for (int i = 0; i < size; i++) {
if (isCancelled()) {
break;
}
publish(new ProgressWindow.ProgressUpdate(i, size, "refreshing file tags", ""));
ContentTag contentTag = contentTags.get(i);
eventDB.addTag(contentTag.getContent().getId(), null, contentTag);
}
LOGGER.log(Level.INFO, "updating artifact tags"); // NON-NLS
List<BlackboardArtifactTag> artifactTags = tagsManager.getAllBlackboardArtifactTags();
size = artifactTags.size();
for (int i = 0; i < size; i++) {
if (isCancelled()) {
break;
}
publish(new ProgressWindow.ProgressUpdate(i, size, "refreshing result tags", ""));
BlackboardArtifactTag artifactTag = artifactTags.get(i);
eventDB.addTag(artifactTag.getContent().getId(), artifactTag.getArtifact().getArtifactID(), artifactTag);
}
LOGGER.log(Level.INFO, "committing tags"); // NON-NLS
publish(new ProgressWindow.ProgressUpdate(0, -1, "committing tag changes", ""));
if (isCancelled()) {
eventDB.rollBackTransaction(trans);
} else {
eventDB.commitTransaction(trans);
}
populateFilterData(skCase);
invalidateCaches();
return null;
}
/**
* handle intermediate 'results': just update progress dialog
*
* @param chunks
*/
@Override
protected void process(List<ProgressWindow.ProgressUpdate> chunks) {
super.process(chunks);
ProgressWindow.ProgressUpdate chunk = chunks.get(chunks.size() - 1);
progressDialog.update(chunk);
}
@Override
@NbBundle.Messages("msgdlg.tagsproblem.text=There was a problem refreshing the tagged events."
+ " Some events may have inacurate tags. See the log for details.")
protected void done() {
super.done();
try {
progressDialog.close();
get();
} catch (CancellationException ex) {
LOGGER.log(Level.INFO, "Database population was cancelled by the user. Not all events may be present or accurate. See the log for details.", ex); // NON-NLS
} catch (InterruptedException | ExecutionException ex) {
LOGGER.log(Level.WARNING, "Exception while populating database.", ex); // NON-NLS
JOptionPane.showMessageDialog(null, Bundle.msgdlg_tagsproblem_text());
} catch (Exception ex) {
LOGGER.log(Level.WARNING, "Unexpected exception while populating database.", ex); // NON-NLS
JOptionPane.showMessageDialog(null, Bundle.msgdlg_tagsproblem_text());
}
postPopulationOperation.run(); //execute post db population operation
}
}
private class DBPopulationWorker extends SwingWorker<Void, ProgressWindow.ProgressUpdate> {
private final ProgressWindow progressDialog;
@@ -268,7 +378,7 @@ public class EventsRepository {
"progressWindow.msg.reinit_db=(re)initializing events database",
"progressWindow.msg.commitingDb=committing events db"})
protected Void doInBackground() throws Exception {
process(Arrays.asList(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_reinit_db(), "")));
publish(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_reinit_db(), ""));
//reset database
//TODO: can we do more incremental updates? -jm
eventDB.reInitializeDB();
@@ -277,7 +387,7 @@ public class EventsRepository {
List<Long> files = skCase.findAllFileIdsWhere("name != '.' AND name != '..'");
final int numFiles = files.size();
process(Arrays.asList(new ProgressWindow.ProgressUpdate(0, numFiles, Bundle.progressWindow_msg_populateMacEventsFiles(), "")));
publish(new ProgressWindow.ProgressUpdate(0, numFiles, Bundle.progressWindow_msg_populateMacEventsFiles(), ""));
//insert file events into db
int i = 1;
@@ -319,8 +429,8 @@ public class EventsRepository {
eventDB.insertEvent(f.getCrtime(), FileSystemTypes.FILE_CREATED, datasourceID, fID, null, uniquePath, medD, shortDesc, known, hashSets, tags, trans);
}
process(Arrays.asList(new ProgressWindow.ProgressUpdate(i, numFiles,
Bundle.progressWindow_msg_populateMacEventsFiles(), f.getName())));
publish(new ProgressWindow.ProgressUpdate(i, numFiles,
Bundle.progressWindow_msg_populateMacEventsFiles(), f.getName()));
}
} catch (TskCoreException tskCoreException) {
LOGGER.log(Level.WARNING, "failed to insert mac event for file : " + fID, tskCoreException); // NON-NLS
@@ -341,11 +451,11 @@ public class EventsRepository {
}
}
process(Arrays.asList(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_commitingDb(), "")));
publish(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_commitingDb(), ""));
if (isCancelled()) {
eventDB.rollBackTransaction(trans);
} else {
eventDB.commitTransaction(trans, true);
eventDB.commitTransaction(trans);
}
populateFilterData(skCase);
@@ -450,6 +560,7 @@ public class EventsRepository {
}
try {
//should this only be tags applied to files or event bearing artifacts?
tagNames.setAll(skCase.getTagNamesInUse());
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Failed to get tag names in use.", ex);

10
Core/src/org/sleuthkit/autopsy/timeline/ui/VisualizationPanel.java
View File

@@ -81,6 +81,7 @@ import org.sleuthkit.autopsy.timeline.actions.SaveSnapshot;
import org.sleuthkit.autopsy.timeline.actions.ZoomOut;
import org.sleuthkit.autopsy.timeline.datamodel.FilteredEventsModel;
import org.sleuthkit.autopsy.timeline.events.TagsUpdatedEvent;
import org.sleuthkit.autopsy.timeline.filters.TagsFilter;
import static org.sleuthkit.autopsy.timeline.ui.Bundle.VisualizationPanel_refresh;
import static org.sleuthkit.autopsy.timeline.ui.Bundle.VisualizationPanel_tagsAddedOrDeleted;
import org.sleuthkit.autopsy.timeline.ui.countsview.CountsViewPane;
@@ -398,9 +399,12 @@ public class VisualizationPanel extends BorderPane implements TimeLineView {
@Subscribe
@NbBundle.Messages("VisualizationPanel.tagsAddedOrDeleted=Tags have been created and/or deleted. The visualization may not be up to date.")
public void handleTimeLineTagEvent(TagsUpdatedEvent event) {
Platform.runLater(() -> {
notificationPane.show(VisualizationPanel_tagsAddedOrDeleted(), new ImageView(INFORMATION));
});
TagsFilter tagsFilter = filteredEvents.getFilter().getTagsFilter();
if (tagsFilter.isSelected() && tagsFilter.isDisabled() == false) {
Platform.runLater(() -> {
notificationPane.show(VisualizationPanel_tagsAddedOrDeleted(), new ImageView(INFORMATION));
});
}
}
synchronized private void refreshHistorgram() {

45
Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AggregateEventNode.java
View File

@@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.timeline.ui.detailview;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
@@ -73,10 +72,6 @@ import org.sleuthkit.autopsy.timeline.filters.TextFilter;
import org.sleuthkit.autopsy.timeline.filters.TypeFilter;
import org.sleuthkit.autopsy.timeline.zooming.DescriptionLOD;
import org.sleuthkit.autopsy.timeline.zooming.ZoomParams;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifactTag;
import org.sleuthkit.datamodel.ContentTag;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
@@ -301,27 +296,25 @@ public class AggregateEventNode extends StackPane {
Map<String, Long> tagCounts = new HashMap<>();
if (!aggEvent.getEventIDsWithTags().isEmpty()) {
try {
for (TimeLineEvent tle : eventsModel.getEventsById(aggEvent.getEventIDsWithTags())) {
AbstractFile abstractFileById = sleuthkitCase.getAbstractFileById(tle.getFileID());
List<ContentTag> contentTags = sleuthkitCase.getContentTagsByContent(abstractFileById);
for (ContentTag tag : contentTags) {
tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum);
}
Long artifactID = tle.getArtifactID();
if (Objects.nonNull(artifactID)) {
BlackboardArtifact blackboardArtifact = sleuthkitCase.getBlackboardArtifact(artifactID);
List<BlackboardArtifactTag> artifactTags = sleuthkitCase.getBlackboardArtifactTagsByArtifact(blackboardArtifact);
for (BlackboardArtifactTag tag : artifactTags) {
tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum);
}
}
}
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error getting tag info for event.", ex);
}
tagCounts.putAll( eventsModel.getTagCountsByTagName(aggEvent.getEventIDsWithTags()));
//
// for (TimeLineEvent tle : eventsModel.getEventsById(aggEvent.getEventIDsWithTags())) {
//
// AbstractFile abstractFileById = sleuthkitCase.getAbstractFileById(tle.getFileID());
// List<ContentTag> contentTags = sleuthkitCase.getContentTagsByContent(abstractFileById);
// for (ContentTag tag : contentTags) {
// tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum);
// }
//
// Long artifactID = tle.getArtifactID();
// if (Objects.nonNull(artifactID)) {
// BlackboardArtifact blackboardArtifact = sleuthkitCase.getBlackboardArtifact(artifactID);
// List<BlackboardArtifactTag> artifactTags = sleuthkitCase.getBlackboardArtifactTagsByArtifact(blackboardArtifact);
// for (BlackboardArtifactTag tag : artifactTags) {
// tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum);
// }
// }
// }
}
String hashSetCountsString = hashSetCounts.entrySet().stream()