diff --git a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java index e60bcddbaf..5a453bfcff 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java @@ -297,7 +297,6 @@ public class TimeLineController { //TODO: verify this locking is correct? -jm synchronized (eventsRepository) { eventsRepository.rebuildRepository(() -> { - synchronized (eventsRepository) { eventsRepository.recordLastObjID(lastObjId); eventsRepository.recordLastArtifactID(lastArtfID); @@ -309,7 +308,7 @@ public class TimeLineController { needsHistogramRebuild.set(false); showWindow(); } - + Platform.runLater(() -> { //TODO: should this be an event? newEventsFlag.set(false); @@ -325,6 +324,25 @@ public class TimeLineController { return true; } + boolean rebuildTagsTable() { + + LOGGER.log(Level.INFO, "starting to rebuild tags table"); // NON-NLS + SwingUtilities.invokeLater(() -> { + if (isWindowOpen()) { + mainFrame.close(); + } + }); + synchronized (eventsRepository) { + eventsRepository.rebuildTags(() -> { + showWindow(); + Platform.runLater(() -> { + showFullRange(); + }); + }); + } + return true; + } + public void showFullRange() { synchronized (filteredEvents) { pushTimeRange(filteredEvents.getSpanningInterval()); @@ -389,14 +407,12 @@ public class TimeLineController { } } } - /* - * if the repo was not rebuilt show the UI. If the repo was rebuild - * it will be displayed as part of that process + * if the repo was not rebuilt at minimum rebuild the tags which may + * have been updated without or knowing it. */ if (repoRebuilt == false) { - showWindow(); - showFullRange(); + rebuildTagsTable(); } } catch (TskCoreException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java index 337c9e30b0..3d44120711 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java @@ -242,6 +242,10 @@ public final class FilteredEventsModel { return repo.getEventsById(eventIDs); } + public Map getTagCountsByTagName(Set eventIDsWithTags) { + return repo.getTagCountsByTagName(eventIDsWithTags); + } + public Set getEventIDs(Interval timeRange, Filter filter) { final Interval overlap; final RootFilter intersect; @@ -398,4 +402,5 @@ public final class FilteredEventsModel { public void refresh() { eventbus.post(new RefreshRequestedEvent()); } + } diff --git a/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java b/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java index e600ddc080..620ff8c91f 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java @@ -155,6 +155,7 @@ public class EventDB { private PreparedStatement dropEventsTableStmt; private PreparedStatement dropHashSetHitsTableStmt; private PreparedStatement dropHashSetsTableStmt; + private PreparedStatement dropTagsTableStmt; private PreparedStatement dropDBInfoTableStmt; private PreparedStatement selectEventIDsFromOBjectAndArtifactStmt; @@ -208,11 +209,11 @@ public class EventDB { return new EventTransaction(); } - void commitTransaction(EventTransaction tr, Boolean notify) { + void commitTransaction(EventTransaction tr) { if (tr.isClosed()) { throw new IllegalArgumentException("can't close already closed transaction"); // NON-NLS } - tr.commit(notify); + tr.commit(); } /** @@ -252,6 +253,25 @@ public class EventDB { } } + Map getTagCountsByTagName(Set eventIDsWithTags) { + HashMap counts = new HashMap<>(); + try (Statement createStatement = con.createStatement(); + ResultSet rs = createStatement.executeQuery("SELECT tag_name_displayName, COUNT(DISTINCT tag_id) AS count FROM tags" + + " WHERE event_id IN (" + StringUtils.join(eventIDsWithTags, ", ") + ")" + + " GROUP BY tag_name_id" + + " ORDER BY tag_name_displayName");) { + while (rs.next()) { + counts.put(rs.getString("tag_name_displayName"), rs.getLong("count")); + } + + } catch (SQLException ex) { + Exceptions.printStackTrace(ex); + } finally { + + } + return counts; + } + /** * drop the tables from this database and recreate them in order to start * over. @@ -262,10 +282,23 @@ public class EventDB { dropEventsTableStmt.executeUpdate(); dropHashSetHitsTableStmt.executeUpdate(); dropHashSetsTableStmt.executeUpdate(); + dropTagsTableStmt.executeUpdate(); dropDBInfoTableStmt.executeUpdate(); - initializeDB();; + initializeDB(); } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "could not drop old tables table", ex); // NON-NLS + LOGGER.log(Level.SEVERE, "could not drop old tables", ex); // NON-NLS + } finally { + DBLock.unlock(); + } + } + + void reInitializeTags() { + DBLock.lock(); + try { + dropTagsTableStmt.executeUpdate(); + initializeTagsTable(); + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "could not drop old tags table", ex); // NON-NLS } finally { DBLock.unlock(); } @@ -535,16 +568,8 @@ public class EventDB { } catch (SQLException ex) { LOGGER.log(Level.SEVERE, "problem creating hash_set_hits table", ex); } - try (Statement stmt = con.createStatement()) { - String sql = "CREATE TABLE IF NOT EXISTS tags " - + "(tag_id INTEGER NOT NULL," - + " tag_name_id INTEGER NOT NULL, " - + " event_id INTEGER REFERENCES events(event_id) NOT NULL, " - + " PRIMARY KEY (event_id, tag_id))"; - stmt.execute(sql); - } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "problem creating hash_set_hits table", ex); - } + + initializeTagsTable(); createIndex("events", Arrays.asList("file_id")); createIndex("events", Arrays.asList("artifact_id")); @@ -567,12 +592,13 @@ public class EventDB { insertHashSetStmt = prepareStatement("INSERT OR IGNORE INTO hash_sets (hash_set_name) values (?)"); selectHashSetStmt = prepareStatement("SELECT hash_set_id FROM hash_sets WHERE hash_set_name = ?"); insertHashHitStmt = prepareStatement("INSERT OR IGNORE INTO hash_set_hits (hash_set_id, event_id) values (?,?)"); - insertTagStmt = prepareStatement("INSERT OR IGNORE INTO tags (tag_id, tag_name_id, event_id) values (?,?,?)"); + insertTagStmt = prepareStatement("INSERT OR IGNORE INTO tags (tag_id, tag_name_id,tag_name_displayName, event_id) values (?,?,?,?)"); deleteTagStmt = prepareStatement("DELETE FROM tags WHERE tag_id = ?"); countAllEventsStmt = prepareStatement("SELECT count(*) AS count FROM events"); dropEventsTableStmt = prepareStatement("DROP TABLE IF EXISTS events"); dropHashSetHitsTableStmt = prepareStatement("DROP TABLE IF EXISTS hash_set_hits"); dropHashSetsTableStmt = prepareStatement("DROP TABLE IF EXISTS hash_sets"); + dropTagsTableStmt = prepareStatement("DROP TABLE IF EXISTS tags"); dropDBInfoTableStmt = prepareStatement("DROP TABLE IF EXISTS db_ino"); selectEventIDsFromOBjectAndArtifactStmt = prepareStatement("SELECT event_id FROM events WHERE file_id == ? AND artifact_id IS ?"); } catch (SQLException sQLException) { @@ -583,6 +609,20 @@ public class EventDB { } } + private void initializeTagsTable() { + try (Statement stmt = con.createStatement()) { + String sql = "CREATE TABLE IF NOT EXISTS tags " + + "(tag_id INTEGER NOT NULL," + + " tag_name_id INTEGER NOT NULL, " + + " tag_name_displayName TEXT NOT NULL, " + + " event_id INTEGER REFERENCES events(event_id) NOT NULL, " + + " PRIMARY KEY (event_id, tag_name_id))"; + stmt.execute(sql); + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "problem creating hash_set_hits table", ex); + } + } + /** * * @param tableName the value of tableName @@ -638,7 +678,7 @@ public class EventDB { EventTransaction transaction = beginTransaction(); insertEvent(time, type, datasourceID, objID, artifactID, fullDescription, medDescription, shortDescription, known, hashSets, tags, transaction); - commitTransaction(transaction, true); + commitTransaction(transaction); } /** @@ -714,11 +754,8 @@ public class EventDB { } } for (Tag tag : tags) { - //"INSERT OR IGNORE INTO tags (tag_id, tag_name_id, event_id) values (?,?,?)"); - insertTagStmt.setLong(1, tag.getId()); - insertTagStmt.setLong(2, tag.getName().getId()); - insertTagStmt.setLong(3, eventID); - insertTagStmt.executeUpdate(); + //could this be one insert? is there a performance win? + insertTag(tag, eventID); } break; } @@ -736,13 +773,7 @@ public class EventDB { try { Set eventIDs = markEventsTagged(objectID, artifactID, true); for (Long eventID : eventIDs) { - //could this be one insert? is there a performance win? - //"INSERT OR IGNORE INTO tags (tag_id, tag_name_id, event_id) values (?,?,?)" - insertTagStmt.clearParameters(); - insertTagStmt.setLong(1, tag.getId()); - insertTagStmt.setLong(2, tag.getName().getId()); - insertTagStmt.setLong(3, eventID); - insertTagStmt.executeUpdate(); + insertTag(tag, eventID); } return eventIDs; } catch (SQLException ex) { @@ -753,6 +784,17 @@ public class EventDB { return Collections.emptySet(); } + private void insertTag(Tag tag, Long eventID) throws SQLException { + + //"INSERT OR IGNORE INTO tags (tag_id, tag_name_id,tag_name_displayName, event_id) values (?,?,?,?)" + insertTagStmt.clearParameters(); + insertTagStmt.setLong(1, tag.getId()); + insertTagStmt.setLong(2, tag.getName().getId()); + insertTagStmt.setString(3, tag.getName().getDisplayName()); + insertTagStmt.setLong(4, eventID); + insertTagStmt.executeUpdate(); + } + Set deleteTag(long objectID, Long artifactID, Tag tag, boolean stillTagged) { DBLock.lock(); try { @@ -1177,16 +1219,13 @@ public class EventDB { } } - private void commit(Boolean notify) { + private void commit() { if (!closed) { try { con.commit(); // make sure we close before we update, bc they'll need locks close(); - if (notify) { -// fireNewEvents(newEvents); - } } catch (SQLException ex) { LOGGER.log(Level.SEVERE, "Error commiting events.db.", ex); // NON-NLS rollback(); diff --git a/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java b/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java index e590c68038..b0f5cedaa8 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java @@ -22,7 +22,6 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; @@ -239,10 +238,121 @@ public class EventsRepository { dbPopulationWorker.execute(); } + synchronized public void rebuildTags(Runnable r) { + if (dbPopulationWorker != null) { + dbPopulationWorker.cancel(true); + + } + dbPopulationWorker = new RebuildTagsWorker(r); + dbPopulationWorker.execute(); + } + public boolean hasDataSourceInfo() { return eventDB.hasNewColumns(); } + public Map getTagCountsByTagName(Set eventIDsWithTags) { + return eventDB.getTagCountsByTagName(eventIDsWithTags); + } + + private class RebuildTagsWorker extends SwingWorker { + + private final ProgressWindow progressDialog; + + //TODO: can we avoid this with a state listener? does it amount to the same thing? + //post population operation to execute + private final Runnable postPopulationOperation; + private final SleuthkitCase skCase; + private final TagsManager tagsManager; + + public RebuildTagsWorker(Runnable postPopulationOperation) { + progressDialog = new ProgressWindow(null, true, this); + progressDialog.setVisible(true); + + skCase = autoCase.getSleuthkitCase(); + tagsManager = autoCase.getServices().getTagsManager(); + + this.postPopulationOperation = postPopulationOperation; + } + + @Override + protected Void doInBackground() throws Exception { + + EventDB.EventTransaction trans = eventDB.beginTransaction(); + LOGGER.log(Level.INFO, "dropping old tags"); // NON-NLS + eventDB.reInitializeTags(); + + LOGGER.log(Level.INFO, "updating content tags"); // NON-NLS + List contentTags = tagsManager.getAllContentTags(); + int size = contentTags.size(); + for (int i = 0; i < size; i++) { + if (isCancelled()) { + break; + } + publish(new ProgressWindow.ProgressUpdate(i, size, "refreshing file tags", "")); + ContentTag contentTag = contentTags.get(i); + eventDB.addTag(contentTag.getContent().getId(), null, contentTag); + } + LOGGER.log(Level.INFO, "updating artifact tags"); // NON-NLS + List artifactTags = tagsManager.getAllBlackboardArtifactTags(); + size = artifactTags.size(); + for (int i = 0; i < size; i++) { + if (isCancelled()) { + break; + } + publish(new ProgressWindow.ProgressUpdate(i, size, "refreshing result tags", "")); + BlackboardArtifactTag artifactTag = artifactTags.get(i); + eventDB.addTag(artifactTag.getContent().getId(), artifactTag.getArtifact().getArtifactID(), artifactTag); + } + + LOGGER.log(Level.INFO, "committing tags"); // NON-NLS + publish(new ProgressWindow.ProgressUpdate(0, -1, "committing tag changes", "")); + if (isCancelled()) { + eventDB.rollBackTransaction(trans); + } else { + eventDB.commitTransaction(trans); + } + + populateFilterData(skCase); + invalidateCaches(); + + return null; + } + + /** + * handle intermediate 'results': just update progress dialog + * + * @param chunks + */ + @Override + protected void process(List chunks) { + super.process(chunks); + ProgressWindow.ProgressUpdate chunk = chunks.get(chunks.size() - 1); + progressDialog.update(chunk); + } + + @Override + @NbBundle.Messages("msgdlg.tagsproblem.text=There was a problem refreshing the tagged events." + + " Some events may have inacurate tags. See the log for details.") + protected void done() { + super.done(); + try { + progressDialog.close(); + get(); + } catch (CancellationException ex) { + LOGGER.log(Level.INFO, "Database population was cancelled by the user. Not all events may be present or accurate. See the log for details.", ex); // NON-NLS + } catch (InterruptedException | ExecutionException ex) { + LOGGER.log(Level.WARNING, "Exception while populating database.", ex); // NON-NLS + JOptionPane.showMessageDialog(null, Bundle.msgdlg_tagsproblem_text()); + } catch (Exception ex) { + LOGGER.log(Level.WARNING, "Unexpected exception while populating database.", ex); // NON-NLS + JOptionPane.showMessageDialog(null, Bundle.msgdlg_tagsproblem_text()); + } + postPopulationOperation.run(); //execute post db population operation + } + + } + private class DBPopulationWorker extends SwingWorker { private final ProgressWindow progressDialog; @@ -268,7 +378,7 @@ public class EventsRepository { "progressWindow.msg.reinit_db=(re)initializing events database", "progressWindow.msg.commitingDb=committing events db"}) protected Void doInBackground() throws Exception { - process(Arrays.asList(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_reinit_db(), ""))); + publish(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_reinit_db(), "")); //reset database //TODO: can we do more incremental updates? -jm eventDB.reInitializeDB(); @@ -277,7 +387,7 @@ public class EventsRepository { List files = skCase.findAllFileIdsWhere("name != '.' AND name != '..'"); final int numFiles = files.size(); - process(Arrays.asList(new ProgressWindow.ProgressUpdate(0, numFiles, Bundle.progressWindow_msg_populateMacEventsFiles(), ""))); + publish(new ProgressWindow.ProgressUpdate(0, numFiles, Bundle.progressWindow_msg_populateMacEventsFiles(), "")); //insert file events into db int i = 1; @@ -319,8 +429,8 @@ public class EventsRepository { eventDB.insertEvent(f.getCrtime(), FileSystemTypes.FILE_CREATED, datasourceID, fID, null, uniquePath, medD, shortDesc, known, hashSets, tags, trans); } - process(Arrays.asList(new ProgressWindow.ProgressUpdate(i, numFiles, - Bundle.progressWindow_msg_populateMacEventsFiles(), f.getName()))); + publish(new ProgressWindow.ProgressUpdate(i, numFiles, + Bundle.progressWindow_msg_populateMacEventsFiles(), f.getName())); } } catch (TskCoreException tskCoreException) { LOGGER.log(Level.WARNING, "failed to insert mac event for file : " + fID, tskCoreException); // NON-NLS @@ -341,11 +451,11 @@ public class EventsRepository { } } - process(Arrays.asList(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_commitingDb(), ""))); + publish(new ProgressWindow.ProgressUpdate(0, -1, Bundle.progressWindow_msg_commitingDb(), "")); if (isCancelled()) { eventDB.rollBackTransaction(trans); } else { - eventDB.commitTransaction(trans, true); + eventDB.commitTransaction(trans); } populateFilterData(skCase); @@ -450,6 +560,7 @@ public class EventsRepository { } try { + //should this only be tags applied to files or event bearing artifacts? tagNames.setAll(skCase.getTagNamesInUse()); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Failed to get tag names in use.", ex); diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/VisualizationPanel.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/VisualizationPanel.java index cb6a38ecbe..0794636646 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/VisualizationPanel.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/VisualizationPanel.java @@ -81,6 +81,7 @@ import org.sleuthkit.autopsy.timeline.actions.SaveSnapshot; import org.sleuthkit.autopsy.timeline.actions.ZoomOut; import org.sleuthkit.autopsy.timeline.datamodel.FilteredEventsModel; import org.sleuthkit.autopsy.timeline.events.TagsUpdatedEvent; +import org.sleuthkit.autopsy.timeline.filters.TagsFilter; import static org.sleuthkit.autopsy.timeline.ui.Bundle.VisualizationPanel_refresh; import static org.sleuthkit.autopsy.timeline.ui.Bundle.VisualizationPanel_tagsAddedOrDeleted; import org.sleuthkit.autopsy.timeline.ui.countsview.CountsViewPane; @@ -398,9 +399,12 @@ public class VisualizationPanel extends BorderPane implements TimeLineView { @Subscribe @NbBundle.Messages("VisualizationPanel.tagsAddedOrDeleted=Tags have been created and/or deleted. The visualization may not be up to date.") public void handleTimeLineTagEvent(TagsUpdatedEvent event) { - Platform.runLater(() -> { - notificationPane.show(VisualizationPanel_tagsAddedOrDeleted(), new ImageView(INFORMATION)); - }); + TagsFilter tagsFilter = filteredEvents.getFilter().getTagsFilter(); + if (tagsFilter.isSelected() && tagsFilter.isDisabled() == false) { + Platform.runLater(() -> { + notificationPane.show(VisualizationPanel_tagsAddedOrDeleted(), new ImageView(INFORMATION)); + }); + } } synchronized private void refreshHistorgram() { diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AggregateEventNode.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AggregateEventNode.java index 8ae04f3d9f..5383672c6b 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AggregateEventNode.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AggregateEventNode.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.timeline.ui.detailview; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.logging.Level; @@ -73,10 +72,6 @@ import org.sleuthkit.autopsy.timeline.filters.TextFilter; import org.sleuthkit.autopsy.timeline.filters.TypeFilter; import org.sleuthkit.autopsy.timeline.zooming.DescriptionLOD; import org.sleuthkit.autopsy.timeline.zooming.ZoomParams; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardArtifactTag; -import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -301,27 +296,25 @@ public class AggregateEventNode extends StackPane { Map tagCounts = new HashMap<>(); if (!aggEvent.getEventIDsWithTags().isEmpty()) { - try { - for (TimeLineEvent tle : eventsModel.getEventsById(aggEvent.getEventIDsWithTags())) { - - AbstractFile abstractFileById = sleuthkitCase.getAbstractFileById(tle.getFileID()); - List contentTags = sleuthkitCase.getContentTagsByContent(abstractFileById); - for (ContentTag tag : contentTags) { - tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum); - } - - Long artifactID = tle.getArtifactID(); - if (Objects.nonNull(artifactID)) { - BlackboardArtifact blackboardArtifact = sleuthkitCase.getBlackboardArtifact(artifactID); - List artifactTags = sleuthkitCase.getBlackboardArtifactTagsByArtifact(blackboardArtifact); - for (BlackboardArtifactTag tag : artifactTags) { - tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum); - } - } - } - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error getting tag info for event.", ex); - } + tagCounts.putAll( eventsModel.getTagCountsByTagName(aggEvent.getEventIDsWithTags())); +// +// for (TimeLineEvent tle : eventsModel.getEventsById(aggEvent.getEventIDsWithTags())) { +// +// AbstractFile abstractFileById = sleuthkitCase.getAbstractFileById(tle.getFileID()); +// List contentTags = sleuthkitCase.getContentTagsByContent(abstractFileById); +// for (ContentTag tag : contentTags) { +// tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum); +// } +// +// Long artifactID = tle.getArtifactID(); +// if (Objects.nonNull(artifactID)) { +// BlackboardArtifact blackboardArtifact = sleuthkitCase.getBlackboardArtifact(artifactID); +// List artifactTags = sleuthkitCase.getBlackboardArtifactTagsByArtifact(blackboardArtifact); +// for (BlackboardArtifactTag tag : artifactTags) { +// tagCounts.merge(tag.getName().getDisplayName(), 1l, Long::sum); +// } +// } +// } } String hashSetCountsString = hashSetCounts.entrySet().stream()