From 27a33fd103716bb94f628700c2fa3d2af4ce90ae Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 28 Jun 2018 10:22:53 -0400 Subject: [PATCH 01/96] removed hash search action --- .../datamodel/DataModelActionsFactory.java | 2 - .../sleuthkit/autopsy/datamodel/FileNode.java | 2 - .../autopsy/datamodel/LocalFileNode.java | 3 -- .../directorytree/DataResultFilterNode.java | 6 --- .../directorytree/HashSearchAction.java | 52 ------------------- .../keywordsearch/AdHocSearchFilterNode.java | 25 ++++----- 6 files changed, 9 insertions(+), 81 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/directorytree/HashSearchAction.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java index b4b88463c4..a5eb26cab9 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java @@ -34,7 +34,6 @@ import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.datamodel.Reports.ReportNode; import org.sleuthkit.autopsy.directorytree.ExternalViewerAction; import org.sleuthkit.autopsy.directorytree.ExtractAction; -import org.sleuthkit.autopsy.directorytree.HashSearchAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.autopsy.directorytree.ViewContextAction; import org.sleuthkit.datamodel.AbstractFile; @@ -82,7 +81,6 @@ public class DataModelActionsFactory { actionsList.add(new ExternalViewerAction(OPEN_IN_EXTERNAL_VIEWER, fileNode)); actionsList.add(null); // creates a menu separator actionsList.add(ExtractAction.getInstance()); - actionsList.add(new HashSearchAction(SEARCH_FOR_FILES_SAME_MD5, fileNode)); actionsList.add(null); // creates a menu separator actionsList.add(AddContentTagAction.getInstance()); if (isArtifactSource) { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java index 4bb8ced8bd..13e7ef34a5 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java @@ -34,7 +34,6 @@ import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.directorytree.ExternalViewerAction; import org.sleuthkit.autopsy.directorytree.ExtractAction; -import org.sleuthkit.autopsy.directorytree.HashSearchAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.autopsy.directorytree.ViewContextAction; import org.sleuthkit.autopsy.modules.embeddedfileextractor.ExtractArchiveWithPasswordAction; @@ -166,7 +165,6 @@ public class FileNode extends AbstractFsContentNode { actionsList.add(null); // Creates an item separator actionsList.add(ExtractAction.getInstance()); - actionsList.add(new HashSearchAction(Bundle.FileNode_getActions_searchFilesSameMD5_text(), this)); actionsList.add(null); // Creates an item separator actionsList.add(AddContentTagAction.getInstance()); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java index c723506363..823d25a137 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java @@ -36,7 +36,6 @@ import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.directorytree.ExternalViewerAction; import org.sleuthkit.autopsy.directorytree.ExtractAction; -import org.sleuthkit.autopsy.directorytree.HashSearchAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.autopsy.directorytree.ViewContextAction; import org.sleuthkit.autopsy.modules.embeddedfileextractor.ExtractArchiveWithPasswordAction; @@ -107,8 +106,6 @@ public class LocalFileNode extends AbstractAbstractFileNode { actionsList.add(null); // creates a menu separator actionsList.add(ExtractAction.getInstance()); - actionsList.add(new HashSearchAction( - NbBundle.getMessage(this.getClass(), "LocalFileNode.getActions.searchFilesSameMd5.text"), this)); actionsList.add(null); // creates a menu separator actionsList.add(AddContentTagAction.getInstance()); diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java index 3aef1e5a2a..199934d614 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java @@ -398,10 +398,8 @@ public class DataResultFilterNode extends FilterNode { } Content c = ban.getLookup().lookup(File.class); Node n = null; - boolean md5Action = false; if (c != null) { n = new FileNode((AbstractFile) c); - md5Action = true; } else if ((c = ban.getLookup().lookup(Directory.class)) != null) { n = new DirectoryNode((Directory) c); } else if ((c = ban.getLookup().lookup(VirtualDirectory.class)) != null) { @@ -435,10 +433,6 @@ public class DataResultFilterNode extends FilterNode { NbBundle.getMessage(this.getClass(), "DataResultFilterNode.action.openInExtViewer.text"), n)); actionsList.add(null); // creates a menu separator actionsList.add(ExtractAction.getInstance()); - if (md5Action) { - actionsList.add(new HashSearchAction( - NbBundle.getMessage(this.getClass(), "DataResultFilterNode.action.searchFilesSameMd5.text"), n)); - } actionsList.add(null); // creates a menu separator actionsList.add(AddContentTagAction.getInstance()); actionsList.add(AddBlackboardArtifactTagAction.getInstance()); diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/HashSearchAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/HashSearchAction.java deleted file mode 100644 index b7cd2eab12..0000000000 --- a/Core/src/org/sleuthkit/autopsy/directorytree/HashSearchAction.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.directorytree; - -import java.awt.event.ActionEvent; -import javax.annotation.concurrent.Immutable; -import javax.swing.AbstractAction; -import org.openide.nodes.Node; -import org.sleuthkit.autopsy.modules.hashdatabase.HashDbSearchAction; - -/** - * Action to lookup the interface and call the real action in HashDatabase. The - * real action, HashDbSearchAction, implements HashSearchProvider, and should be - * the only instance of it. - * - * //TODO: HashDBSearchAction needs a public constructor and a service - * registration annotation for the lookup technique to work - */ -@Immutable -public class HashSearchAction extends AbstractAction { - - private final Node contentNode; - - public HashSearchAction(String title, Node contentNode) { - super(title); - this.contentNode = contentNode; - } - - @Override - public void actionPerformed(ActionEvent e) { - //HashSearchProvider searcher = Lookup.getDefault().lookup(HashSearchProvider.class); - //TODO: HashDBSearchAction needs a public constructor and a service registration annotation for the above technique to work - HashDbSearchAction searcher = HashDbSearchAction.getDefault(); - searcher.search(contentNode); - } -} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchFilterNode.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchFilterNode.java index 3177bbbbb4..57b84b11d6 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchFilterNode.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchFilterNode.java @@ -36,7 +36,6 @@ import org.sleuthkit.autopsy.directorytree.ExternalViewerAction; import org.sleuthkit.autopsy.directorytree.ExtractAction; import org.sleuthkit.autopsy.actions.AddContentTagAction; import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction; -import org.sleuthkit.autopsy.directorytree.HashSearchAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; @@ -48,7 +47,6 @@ import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.Report; import org.sleuthkit.datamodel.SlackFile; -import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.VirtualDirectory; /** @@ -126,50 +124,45 @@ class AdHocSearchFilterNode extends FilterNode { @Override public List visit(File f) { - return getFileActions(true); + return getFileActions(); } @Override public List visit(DerivedFile f) { - return getFileActions(true); + return getFileActions(); } @Override public List visit(Directory d) { - return getFileActions(false); + return getFileActions(); } @Override public List visit(LayoutFile lf) { - //we want hashsearch enabled on carved files but not unallocated blocks - boolean enableHashSearch = (lf.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.CARVED); - return getFileActions(enableHashSearch); + return getFileActions(); } @Override public List visit(LocalFile lf) { - return getFileActions(true); + return getFileActions(); } @Override public List visit(SlackFile f) { - return getFileActions(false); + return getFileActions(); } @Override public List visit(VirtualDirectory dir) { - return getFileActions(false); + return getFileActions(); } - private List getFileActions(boolean enableHashSearch) { + private List getFileActions() { List actionsList = new ArrayList<>(); actionsList.add(new NewWindowViewAction(NbBundle.getMessage(this.getClass(), "KeywordSearchFilterNode.getFileActions.viewInNewWinActionLbl"), AdHocSearchFilterNode.this)); actionsList.add(new ExternalViewerAction(NbBundle.getMessage(this.getClass(), "KeywordSearchFilterNode.getFileActions.openExternViewActLbl"), getOriginal())); actionsList.add(null); actionsList.add(ExtractAction.getInstance()); - Action hashSearchAction = new HashSearchAction(NbBundle.getMessage(this.getClass(), "KeywordSearchFilterNode.getFileActions.searchSameMd5"), getOriginal()); - hashSearchAction.setEnabled(enableHashSearch); - actionsList.add(hashSearchAction); actionsList.add(null); // creates a menu separator actionsList.add(AddContentTagAction.getInstance()); @@ -185,7 +178,7 @@ class AdHocSearchFilterNode extends FilterNode { @Override protected List defaultVisit(Content c) { - return getFileActions(false); + return getFileActions(); } } } From b4dc29c67e9676bdb88a467555a09a9fe85feb1d Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 28 Jun 2018 11:50:25 -0400 Subject: [PATCH 02/96] sqlite_filechooser suggests file name --- .../autopsy/contentviewers/SQLiteViewer.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index b7a5498bf8..33aa53fcc1 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -56,7 +56,6 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.TimeStampUtils; /** * A file content viewer for SQLite database files. @@ -289,28 +288,22 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { fileChooser.setAcceptAllFileFilterUsed(true); fileChooser.setFileFilter(csvFilter); fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + String defaultFileName = (String) this.tablesDropdownList.getSelectedItem(); + fileChooser.setSelectedFile(new File(defaultFileName)); int choice = fileChooser.showSaveDialog((Component) evt.getSource()); //TODO if (JFileChooser.APPROVE_OPTION == choice) { - boolean overwrite = false; File file = fileChooser.getSelectedFile(); - if (file == null) { - JOptionPane.showMessageDialog(this, - Bundle.SQLiteViewer_csvExport_fileName_empty(), - Bundle.SQLiteViewer_csvExport_title(), - JOptionPane.WARNING_MESSAGE); - return; - } else if (file.exists() && FilenameUtils.getExtension(file.getName()).equalsIgnoreCase("csv")) { + if (file.exists() && FilenameUtils.getExtension(file.getName()).equalsIgnoreCase("csv")) { if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(this, Bundle.SQLiteViewer_csvExport_confirm_msg(), Bundle.SQLiteViewer_csvExport_title(), JOptionPane.YES_NO_OPTION)) { - overwrite = true; } else { return; } } - exportTableToCsv(file, overwrite); + exportTableToCsv(file); } }//GEN-LAST:event_exportCsvButtonActionPerformed @@ -567,9 +560,9 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { "SQLiteViewer.exportTableToCsv.FileName=File name: ", "SQLiteViewer.exportTableToCsv.TableName=Table name: " }) - private void exportTableToCsv(File file, boolean overwrite) { + private void exportTableToCsv(File file) { String tableName = (String) this.tablesDropdownList.getSelectedItem(); - String csvFileSuffix = "_" + tableName + "_" + TimeStampUtils.createTimeStamp() + ".csv"; + FileOutputStream out = null; try ( Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery("SELECT * FROM " + tableName)) { @@ -578,16 +571,15 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { if (Objects.isNull(currentTableRows) || currentTableRows.isEmpty()) { logger.log(Level.INFO, String.format("The table %s is empty. (objId=%d)", tableName, sqliteDbFile.getId())); //NON-NLS } else { - String fileName = file.getName(); File csvFile; - if (overwrite) { + String fileName = file.getName(); + if (FilenameUtils.getExtension(fileName).equalsIgnoreCase("csv")) { csvFile = file; - } else if (FilenameUtils.getExtension(fileName).equalsIgnoreCase("csv")) { - csvFile = new File(file.getParentFile(), FilenameUtils.removeExtension(fileName) + csvFileSuffix); } else { - csvFile = new File(file.toString() + csvFileSuffix); + csvFile = new File(file.toString() + ".csv"); } - FileOutputStream out = new FileOutputStream(csvFile, false); + + out = new FileOutputStream(csvFile, false); out.write((Bundle.SQLiteViewer_exportTableToCsv_FileName() + csvFile.getName() + "\n").getBytes()); out.write((Bundle.SQLiteViewer_exportTableToCsv_TableName() + tableName + "\n").getBytes()); @@ -622,6 +614,14 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } catch (IOException ex) { logger.log(Level.SEVERE, String.format("Failed to export table %s to file '%s'", tableName, file.getName()), ex); //NON-NLS MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_exportTableToCsv_write_errText()); + }finally{ + try { + if (out != null){ + out.close(); + } + } catch (IOException ex) { + logger.log(Level.WARNING,String.format("unable to properly close the %s after writing to it", file.getName()),ex); + } } } From a05c5a7f313df44e011f6f1ce206853a358b682f Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 28 Jun 2018 12:13:59 -0400 Subject: [PATCH 03/96] removed hash search action --- Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties | 1 - Core/src/org/sleuthkit/autopsy/datamodel/Bundle_ja.properties | 1 - .../sleuthkit/autopsy/datamodel/DataModelActionsFactory.java | 4 ---- 3 files changed, 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties index 374c96069b..f45a333ec1 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties @@ -54,7 +54,6 @@ DataModelActionsFactory.srcFileInDir.text=View Source File in Directory DataModelActionsFactory.fileInDir.text=View File in Directory DataModelActionsFactory.viewNewWin.text=View in New Window DataModelActionsFactory.openExtViewer.text=Open in External Viewer -DataModelActionsFactory.srfFileSameMD5.text=Search for files with the same MD5 hash DataSourcesNode.name=Data Sources DataSourcesNode.group_by_datasource.name=Data Source Files DataSourcesNode.createSheet.name.name=Name diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle_ja.properties index 979c4d50cb..e2c331b390 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle_ja.properties @@ -55,7 +55,6 @@ DataModelActionsFactory.srcFileInDir.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u DataModelActionsFactory.fileInDir.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u5185\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u8868\u793a DataModelActionsFactory.viewNewWin.text=\u65b0\u898f\u30a6\u30a3\u30f3\u30c9\u30a6\u306b\u8868\u793a DataModelActionsFactory.openExtViewer.text=\u5916\u90e8\u30d3\u30e5\u30fc\u30a2\u306b\u8868\u793a -DataModelActionsFactory.srfFileSameMD5.text=\u540c\u3058MD5\u30cf\u30c3\u30b7\u30e5\u3092\u6301\u3064\u30d5\u30a1\u30a4\u30eb\u3092\u691c\u7d22 DataSourcesNode.name=\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9 DataSourcesNode.group_by_datasource.name=\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30d5\u30a1\u30a4\u30eb DataSourcesNode.createSheet.name.name=\u540d\u524d diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java index 66f19fc52e..88e90f054d 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DataModelActionsFactory.java @@ -75,8 +75,6 @@ public class DataModelActionsFactory { .getMessage(DataModelActionsFactory.class, "DataModelActionsFactory.viewNewWin.text"); public static final String OPEN_IN_EXTERNAL_VIEWER = NbBundle .getMessage(DataModelActionsFactory.class, "DataModelActionsFactory.openExtViewer.text"); - public static final String SEARCH_FOR_FILES_SAME_MD5 = NbBundle - .getMessage(DataModelActionsFactory.class, "DataModelActionsFactory.srfFileSameMD5.text"); public static List getActions(File file, boolean isArtifactSource) { List actionsList = new ArrayList<>(); @@ -365,7 +363,6 @@ public class DataModelActionsFactory { actionsList.add(new ExternalViewerAction(OPEN_IN_EXTERNAL_VIEWER, tagNode)); actionsList.add(null); // creates a menu separator actionsList.add(ExtractAction.getInstance()); - actionsList.add(new HashSearchAction(SEARCH_FOR_FILES_SAME_MD5, tagNode)); actionsList.add(null); // creates a menu separator actionsList.add(AddContentTagAction.getInstance()); if (isArtifactSource) { @@ -402,7 +399,6 @@ public class DataModelActionsFactory { actionsList.add(new ExternalViewerAction(OPEN_IN_EXTERNAL_VIEWER, tagNode)); actionsList.add(null); // creates a menu separator actionsList.add(ExtractAction.getInstance()); - actionsList.add(new HashSearchAction(SEARCH_FOR_FILES_SAME_MD5, tagNode)); actionsList.add(null); // creates a menu separator actionsList.add(AddContentTagAction.getInstance()); if (isArtifactSource) { From cc3d1d59b830752abb84dc158ac94dfab8d6df94 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 28 Jun 2018 12:52:06 -0400 Subject: [PATCH 04/96] Add size and path to exif artifact display --- .../datamodel/BlackboardArtifactNode.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 235f00cb1b..20bc9e4470 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -312,7 +312,11 @@ public class BlackboardArtifactNode extends AbstractContentNode(NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.fileSize.name"), + NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.fileSize.displayName"), + NO_DESCR, + size)); + sheetSet.put(new NodeProperty<>( + NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.path.name"), + NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.path.displayName"), + NO_DESCR, + path)); + } addTagProperty(sheetSet); From 2c0f856406955a8ffaa303fe3a6d8625cda115a7 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 28 Jun 2018 15:48:04 -0400 Subject: [PATCH 05/96] 3994: Image gallery does not work with Postgres --- .../imagegallery/datamodel/DrawableDB.java | 6 ++++++ .../datamodel/grouping/GroupManager.java | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 51d58c7f1e..7688ccd146 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -1179,6 +1179,12 @@ public final class DrawableDB { * @return the number of files with Cat-0 */ public long getUncategorizedCount(Collection fileIDs) { + + // if the fileset is empty, return count as 0 + if (fileIDs.isEmpty()) { + return 0; + } + DrawableTagsManager tagsManager = controller.getTagsManager(); // get a comma seperated list of TagName ids for non zero categories diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 7dddfa5ca8..de9c1f6b4f 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -82,6 +82,7 @@ import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData.DbType; /** * Provides an abstraction layer on top of {@link DrawableDB} ( and to some @@ -351,11 +352,22 @@ public class GroupManager { case MIME_TYPE: if (nonNull(db)) { HashSet types = new HashSet<>(); - try (SleuthkitCase.CaseDbQuery executeQuery = controller.getSleuthKitCase().executeQuery("select group_concat(obj_id), mime_type from tsk_files group by mime_type "); //NON-NLS + + // Use the group_concat function to get a list of files for each mime type. + // This has different syntax on Postgres vs SQLite + String groupConcatClause; + if (DbType.POSTGRESQL == controller.getSleuthKitCase().getDatabaseType()) { + groupConcatClause = " array_to_string(array_agg(obj_id), ',') as object_ids"; + } + else { + groupConcatClause = "select group_concat(obj_id) as object_ids"; + } + String query = "select " + groupConcatClause + " , mime_type from tsk_files group by mime_type "; + try (SleuthkitCase.CaseDbQuery executeQuery = controller.getSleuthKitCase().executeQuery(query); //NON-NLS ResultSet resultSet = executeQuery.getResultSet();) { while (resultSet.next()) { final String mimeType = resultSet.getString("mime_type"); //NON-NLS - String objIds = resultSet.getString("group_concat(obj_id)"); //NON-NLS + String objIds = resultSet.getString("object_ids"); //NON-NLS Pattern.compile(",").splitAsStream(objIds) .map(Long::valueOf) From 9d842be3daa04e3e56dd5c00e7c23b0016fbe7d3 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 28 Jun 2018 16:22:54 -0400 Subject: [PATCH 06/96] 3978-UnicodeTextExtractor --- .../KeywordSearchIngestModule.java | 17 +++- .../keywordsearch/UnicodeTextExtractor.java | 78 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 83b770bc58..3ed4f76b38 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -28,7 +28,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -92,6 +91,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private boolean startedSearching = false; private List textExtractors; private StringsTextExtractor stringExtractor; + private UnicodeTextExtractor unicodeExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private long jobId; @@ -244,6 +244,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule { stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); + unicodeExtractor = new UnicodeTextExtractor(); + textExtractors = new ArrayList<>(); //order matters, more specific extractors first textExtractors.add(new HtmlTextExtractor()); @@ -343,7 +345,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { textExtractors.clear(); textExtractors = null; stringExtractor = null; - + unicodeExtractor = null; initialized = false; } @@ -568,6 +570,17 @@ public final class KeywordSearchIngestModule implements FileIngestModule { putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } + if ((wasTextAdded == false) && (aFile.getNameExtension().equalsIgnoreCase("txt"))) { + try { + if (Ingester.getDefault().indexText(unicodeExtractor, aFile, context)) { + putIngestStatus(jobId, aFile.getId(), IngestStatus.TEXT_INGESTED); + wasTextAdded = true; + } + } catch (IngesterException ex) { + logger.log(Level.WARNING, "Unable to index as unicode", ex); + } + } + // if it wasn't supported or had an error, default to strings if (wasTextAdded == false) { extractStringsAndIndex(aFile); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java new file mode 100644 index 0000000000..a2189b8f4c --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java @@ -0,0 +1,78 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.logging.Level; +import org.apache.tika.parser.txt.CharsetDetector; +import org.apache.tika.parser.txt.CharsetMatch; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.ReadContentInputStream; + +/** + * Extract text from unicode files + */ +final class UnicodeTextExtractor extends ContentTextExtractor { + + static final private Logger logger = Logger.getLogger(UnicodeTextExtractor.class.getName()); + + @Override + boolean isContentTypeSpecific() { + return true; + } + + @Override + boolean isSupported(Content file, String detectedFormat) { + return true; + } + + @Override + public Reader getReader(Content source) throws TextExtractorException { + CharsetDetector detector = new CharsetDetector(); + ReadContentInputStream stream = new ReadContentInputStream(source); + byte[] byteData = new byte[(int) source.getSize()]; + try { + stream.read(byteData, 0, (int) source.getSize()); + detector.setText(byteData); + CharsetMatch match = detector.detect(); + try { + return new StringReader(match.getString()); + + } catch (IOException ex) { + throw new TextExtractorException("Unable to get string from detected text in UnicodeTextExtractor", ex); + } + } catch (ReadContentInputStream.ReadContentInputStreamException ex) { + throw new TextExtractorException("Unable to read text stream in UnicodeTextExtractor", ex); + } + } + + @Override + public boolean isDisabled() { + return false; + } + + @Override + public void logWarning(String msg, Exception ex) { + logger.log(Level.WARNING, msg, ex); + } + +} From bd202f5fbff08d7e7178374ef7816b31d77b9e31 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 28 Jun 2018 16:35:16 -0400 Subject: [PATCH 07/96] Revert "Remove redundant data source name tool tips from ad hoc kws" This reverts commit 57d9022c525d077a2b4443a5f1964e031b206df3. --- .../autopsy/keywordsearch/AdHocSearchPanel.java | 11 +++++++++++ .../keywordsearch/DropdownListSearchPanel.java | 16 ++++++++++++++++ .../DropdownSingleTermSearchPanel.java | 16 ++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index 9b84a5c785..c06394bf23 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -43,6 +43,7 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { private final String keywordSearchErrorDialogHeader = org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.dialogErrorHeader"); protected int filesIndexed; private final Map dataSourceMap = new HashMap<>(); + private final List toolTipList = new ArrayList<>(); private List dataSources = new ArrayList<>(); private final DefaultListModel dataSourceListModel = new DefaultListModel<>(); @@ -152,12 +153,14 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { */ synchronized List getDataSourceArray() { List dsList = new ArrayList<>(); + toolTipList.clear(); Collections.sort(this.dataSources, (DataSource ds1, DataSource ds2) -> ds1.getName().compareTo(ds2.getName())); for (DataSource ds : dataSources) { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); dataSourceMap.put(ds.getId(), displayName); + toolTipList.add(dsName); dsList.add(displayName); } return dsList; @@ -180,6 +183,14 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { Map getDataSourceMap() { return dataSourceMap; } + + /** + * Get a list of tooltip text for data source + * @return A list of tool tips + */ + List getDataSourceToolTipList() { + return toolTipList; + } /** * Get a list of DataSourceListModel diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index 17d5921e0e..72a1cbab5a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -73,6 +73,22 @@ class DropdownListSearchPanel extends AdHocSearchPanel { dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); + dataSourceList.addMouseMotionListener(new MouseMotionListener() { + + @Override + public void mouseDragged(MouseEvent evt) { + //Unused by now + } + + @Override + public void mouseMoved(MouseEvent evt) { + JList dsList = (JList) evt.getSource(); + int index = dsList.locationToIndex(evt.getPoint()); + if (index > -1) { + dsList.setToolTipText(getDataSourceToolTipList().get(index)); + } + } + }); } static synchronized DropdownListSearchPanel getDefault() { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java index 06a0bcd8b1..b859df85ef 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java @@ -88,6 +88,22 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); + this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { + + @Override + public void mouseDragged(MouseEvent evt) { + //Unused by now + } + + @Override + public void mouseMoved(MouseEvent evt) { + JList DsList = (JList) evt.getSource(); + int index = DsList.locationToIndex(evt.getPoint()); + if (index > -1) { + DsList.setToolTipText(getDataSourceToolTipList().get(index)); + } + } + }); } /** From 3fdd843ac3b42427a41469215f356e5d5e9c419c Mon Sep 17 00:00:00 2001 From: rishwanth1995 Date: Thu, 28 Jun 2018 16:48:59 -0400 Subject: [PATCH 08/96] used try with resources in sqliteviewer --- .../autopsy/contentviewers/SQLiteViewer.java | 58 ++++++++----------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 33aa53fcc1..40a667e3d7 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -562,7 +562,6 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { }) private void exportTableToCsv(File file) { String tableName = (String) this.tablesDropdownList.getSelectedItem(); - FileOutputStream out = null; try ( Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery("SELECT * FROM " + tableName)) { @@ -579,33 +578,34 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { csvFile = new File(file.toString() + ".csv"); } - out = new FileOutputStream(csvFile, false); + try (FileOutputStream out = new FileOutputStream(csvFile, false)) { - out.write((Bundle.SQLiteViewer_exportTableToCsv_FileName() + csvFile.getName() + "\n").getBytes()); - out.write((Bundle.SQLiteViewer_exportTableToCsv_TableName() + tableName + "\n").getBytes()); - // Set up the column names - Map row = currentTableRows.get(0); - StringBuffer header = new StringBuffer(); - for (Map.Entry col : row.entrySet()) { - String colName = col.getKey(); - if (header.length() > 0) { - header.append(',').append(colName); - } else { - header.append(colName); - } - } - out.write(header.append('\n').toString().getBytes()); - - for (Map maps : currentTableRows) { - StringBuffer valueLine = new StringBuffer(); - maps.values().forEach((value) -> { - if (valueLine.length() > 0) { - valueLine.append(',').append(value.toString()); + out.write((Bundle.SQLiteViewer_exportTableToCsv_FileName() + csvFile.getName() + "\n").getBytes()); + out.write((Bundle.SQLiteViewer_exportTableToCsv_TableName() + tableName + "\n").getBytes()); + // Set up the column names + Map row = currentTableRows.get(0); + StringBuffer header = new StringBuffer(); + for (Map.Entry col : row.entrySet()) { + String colName = col.getKey(); + if (header.length() > 0) { + header.append(',').append(colName); } else { - valueLine.append(value.toString()); + header.append(colName); } - }); - out.write(valueLine.append('\n').toString().getBytes()); + } + out.write(header.append('\n').toString().getBytes()); + + for (Map maps : currentTableRows) { + StringBuffer valueLine = new StringBuffer(); + maps.values().forEach((value) -> { + if (valueLine.length() > 0) { + valueLine.append(',').append(value.toString()); + } else { + valueLine.append(value.toString()); + } + }); + out.write(valueLine.append('\n').toString().getBytes()); + } } } } catch (SQLException ex) { @@ -614,14 +614,6 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } catch (IOException ex) { logger.log(Level.SEVERE, String.format("Failed to export table %s to file '%s'", tableName, file.getName()), ex); //NON-NLS MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_exportTableToCsv_write_errText()); - }finally{ - try { - if (out != null){ - out.close(); - } - } catch (IOException ex) { - logger.log(Level.WARNING,String.format("unable to properly close the %s after writing to it", file.getName()),ex); - } } } From c753ed74e15be6833fd7eb2497450a6f7086c5fa Mon Sep 17 00:00:00 2001 From: esaunders Date: Thu, 28 Jun 2018 17:30:32 -0400 Subject: [PATCH 09/96] Make CAT-0 a standard tag name so that it cannot be deleted and will not show up at the top of the tags list. --- .../autopsy/casemodule/services/TagNameDefinition.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java index fb02e7d2d4..33e8f29237 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java @@ -54,7 +54,7 @@ final class TagNameDefinition implements Comparable { private static final List STANDARD_TAG_DISPLAY_NAMES = Arrays.asList(Bundle.TagNameDefinition_predefTagNames_bookmark_text(), Bundle.TagNameDefinition_predefTagNames_followUp_text(), Bundle.TagNameDefinition_predefTagNames_notableItem_text(), DhsImageCategory.ONE.getDisplayName(), DhsImageCategory.TWO.getDisplayName(), DhsImageCategory.THREE.getDisplayName(), - DhsImageCategory.FOUR.getDisplayName(), DhsImageCategory.FIVE.getDisplayName()); + DhsImageCategory.FOUR.getDisplayName(), DhsImageCategory.FIVE.getDisplayName(), DhsImageCategory.ZERO.getDisplayName()); private final String displayName; private final String description; private final TagName.HTML_COLOR color; From 1ce90afe5931e6a4c2c80b9c93909b4fb5228493 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 15:32:57 -0600 Subject: [PATCH 10/96] added finally block in case anything fails during search the progress bar will still stop --- .../sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java index 356056a1bc..0e94e561b2 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java @@ -264,7 +264,6 @@ public final class CommonFilesPanel extends javax.swing.JPanel { viewers.add(table); progress.setDisplayName(Bundle.CommonFilesPanel_search_done_searchProgressDisplay()); DataResultTopComponent.createInstance(tabTitle, pathText, tableFilterWithDescendantsNode, metadata.size(), viewers); - progress.finish(); } catch (InterruptedException ex) { LOGGER.log(Level.SEVERE, "Interrupted while loading Common Files", ex); MessageNotifyUtil.Message.error(Bundle.CommonFilesPanel_search_done_interupted()); @@ -285,6 +284,8 @@ public final class CommonFilesPanel extends javax.swing.JPanel { errorMessage = Bundle.CommonFilesPanel_search_done_exception(); } MessageNotifyUtil.Message.error(errorMessage); + } finally { + progress.finish(); } } }.execute(); From 0afee2fbf6d8f94f20970c11d8d22b81f06a4227 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 28 Jun 2018 18:55:15 -0400 Subject: [PATCH 11/96] 3994: Image gallery doesnt work for Postgres - Fixed sql for SQLite. --- .../autopsy/imagegallery/datamodel/grouping/GroupManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index de9c1f6b4f..fb381160f7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -360,7 +360,7 @@ public class GroupManager { groupConcatClause = " array_to_string(array_agg(obj_id), ',') as object_ids"; } else { - groupConcatClause = "select group_concat(obj_id) as object_ids"; + groupConcatClause = " group_concat(obj_id) as object_ids"; } String query = "select " + groupConcatClause + " , mime_type from tsk_files group by mime_type "; try (SleuthkitCase.CaseDbQuery executeQuery = controller.getSleuthKitCase().executeQuery(query); //NON-NLS From e40eb48182f5a8f55397cda013fa2a4a13d60753 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 19:06:14 -0600 Subject: [PATCH 12/96] added tests for revised intra case correlation algorithm --- .../commonfilessearch/IntraCaseUtils.java | 28 +++- .../MatchesInAtLeastTwoSources.java | 127 ++++++++++++++++++ 2 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java index 9fddc6b5e1..f81e561b38 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java @@ -104,16 +104,36 @@ class IntraCaseUtils { } void setUp() { - CaseUtils.createAsCurrentCase(this.caseName); + this.createAsCurrentCase(); final ImageDSProcessor imageDSProcessor = new ImageDSProcessor(); - IngestUtils.addDataSource(imageDSProcessor, imagePath1); - IngestUtils.addDataSource(imageDSProcessor, imagePath2); - IngestUtils.addDataSource(imageDSProcessor, imagePath3); + this.addImageOne(imageDSProcessor); + this.addImageTwo(imageDSProcessor); + this.addImageThree(imageDSProcessor); + this.addImageFour(imageDSProcessor); + } + + void addImageFour(final ImageDSProcessor imageDSProcessor) { IngestUtils.addDataSource(imageDSProcessor, imagePath4); } + void addImageThree(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath3); + } + + void addImageTwo(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath2); + } + + void addImageOne(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath1); + } + + void createAsCurrentCase() { + CaseUtils.createAsCurrentCase(this.caseName); + } + Map getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException { return this.dataSourceLoader.getDataSourceMap(); } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java new file mode 100644 index 0000000000..46394edbab --- /dev/null +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java @@ -0,0 +1,127 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilessearch; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import junit.framework.Test; +import org.netbeans.junit.NbModuleSuite; +import org.netbeans.junit.NbTestCase; +import org.openide.util.Exceptions; +import org.python.icu.impl.Assert; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.ImageDSProcessor; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.DOC; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.EMPTY; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.IMG; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET1; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET4; +import org.sleuthkit.autopsy.ingest.IngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleTemplate; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory; +import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory; +import org.sleuthkit.autopsy.testutils.IngestUtils; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * + * @author bsweeney + */ +public class MatchesInAtLeastTwoSources extends NbTestCase { + + public static Test suite() { + NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(MatchesInAtLeastTwoSources.class). + clusters(".*"). + enableModules(".*"); + return conf.suite(); + } + + private final IntraCaseUtils utils; + + public MatchesInAtLeastTwoSources(String name) { + super(name); + + this.utils = new IntraCaseUtils(this, "MatchesInAtLeastTwoSources"); + } + + @Override + public void setUp() { + this.utils.createAsCurrentCase(); + + final ImageDSProcessor imageDSProcessor = new ImageDSProcessor(); + + this.utils.addImageOne(imageDSProcessor); + this.utils.addImageFour(imageDSProcessor); + + IngestModuleTemplate hashLookupTemplate = IngestUtils.getIngestModuleTemplate(new HashLookupModuleFactory()); + IngestModuleTemplate mimeTypeLookupTemplate = IngestUtils.getIngestModuleTemplate(new FileTypeIdModuleFactory()); + + ArrayList templates = new ArrayList<>(); + templates.add(hashLookupTemplate); + templates.add(mimeTypeLookupTemplate); + + IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestedWithHashAndFileType.class.getCanonicalName(), IngestJobSettings.IngestType.FILES_ONLY, templates); + + try { + IngestUtils.runIngestJob(Case.getCurrentCaseThrows().getDataSources(), ingestJobSettings); + } catch (NoCurrentCaseException | TskCoreException ex) { + Exceptions.printStackTrace(ex); + Assert.fail(ex); + } + } + + @Override + public void tearDown() { + this.utils.tearDown(); + } + + public void testOne() { + try { + Map dataSources = this.utils.getDataSourceMap(); + + CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false); + CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles(); + + Map objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata); + + List files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet()); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET4, 0)); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET4, 0)); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET4, 0)); + + } catch (NoCurrentCaseException | TskCoreException | SQLException ex) { + Exceptions.printStackTrace(ex); + Assert.fail(ex); + } + } +} From 7168cfe187c61aa76bbdcee6e9ed2be35948c57e Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 19:08:26 -0600 Subject: [PATCH 13/96] comments --- .../commonfilessearch/MatchesInAtLeastTwoSources.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java index 46394edbab..d45b9a8dc2 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java @@ -48,8 +48,12 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * - * @author bsweeney + * Ensures that matches only are found for files which appear in at least two data sources. + * + * The two datasources used here have no common files. One of the data sources + * has two identical files within it. This should not count as a match. + * + * None of the test files should be found in the results of this test. */ public class MatchesInAtLeastTwoSources extends NbTestCase { From a36c4a61e3ba72d014425e14a9120f86dc5d8558 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 19:27:52 -0600 Subject: [PATCH 14/96] trying to quiet codacy --- .../MatchesInAtLeastTwoSources.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java index d45b9a8dc2..ba178a59f6 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java @@ -34,11 +34,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm; import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.DOC; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.EMPTY; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.IMG; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET1; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET4; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.*; import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleTemplate; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory; @@ -48,11 +44,12 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * Ensures that matches only are found for files which appear in at least two data sources. - * - * The two datasources used here have no common files. One of the data sources - * has two identical files within it. This should not count as a match. - * + * Ensures that matches only are found for files which appear in at least two + * data sources. + * + * The two datasources used here have no common files. One of the data sources + * has two identical files within it. This should not count as a match. + * * None of the test files should be found in the results of this test. */ public class MatchesInAtLeastTwoSources extends NbTestCase { @@ -109,20 +106,20 @@ public class MatchesInAtLeastTwoSources extends NbTestCase { CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false); CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles(); - + Map objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata); List files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet()); - + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET1, 0)); assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET4, 0)); - + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET1, 0)); assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET4, 0)); - + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET1, 0)); assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET4, 0)); - + } catch (NoCurrentCaseException | TskCoreException | SQLException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex); From 4693c6681240f6fde5054c9fb1850d3626a505ab Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 28 Jun 2018 22:39:32 -0400 Subject: [PATCH 15/96] Disable controls when fewer than two data sources available. --- .../DropdownListSearchPanel.java | 22 ++++++++++++++----- .../DropdownSingleTermSearchPanel.java | 18 ++++++++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index 72a1cbab5a..e5c4191201 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -18,7 +18,9 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.awt.*; +import java.awt.Component; +import java.awt.Cursor; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.MouseEvent; @@ -683,11 +685,21 @@ class DropdownListSearchPanel extends AdHocSearchPanel { * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java index b859df85ef..ba872b0e4d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java @@ -390,18 +390,26 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { } setComponentsEnabled(); firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); - } /** * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } From 101192a297f8b85867ccc626424f06de7e043588 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 29 Jun 2018 07:29:45 -0400 Subject: [PATCH 16/96] Fix type for size --- .../sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 20bc9e4470..d6fb9afc04 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -460,11 +460,11 @@ public class BlackboardArtifactNode extends AbstractContentNode Date: Fri, 29 Jun 2018 08:05:39 -0600 Subject: [PATCH 17/96] more logging --- .../CommonFilesSearchAction.java | 1 - .../CommonFilesSearchResultsViewerTable.java | 43 +++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java index 1727c9322b..5a3f887564 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java @@ -24,7 +24,6 @@ import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.core.Installer; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.Logger; diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index 71ffe1a21e..fecb676f4b 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -23,23 +23,27 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; /** - * DataResultViewerTable which overrides the default column - * header width calculations. The CommonFilesSearchResultsViewerTable - * presents multiple tiers of data which are not always present and it may not - * make sense to try to calculate the column widths for such tables by sampling - * rows and looking for wide cells. Rather, we just pick some reasonable values. + * DataResultViewerTable which overrides the default column header + * width calculations. The CommonFilesSearchResultsViewerTable + * presents multiple tiers of data which are not always present and it may not + * make sense to try to calculate the column widths for such tables by sampling + * rows and looking for wide cells. Rather, we just pick some reasonable values. */ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { - + private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; - + + private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); + static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); @@ -49,10 +53,10 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { map.put(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), 100); map.put(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), 130); map.put(Bundle.CommonFilesSearchResultsViewerTable_tagsColLbl1(), 300); - + COLUMN_WIDTHS = Collections.unmodifiableMap(map); } - + @NbBundle.Messages({ "CommonFilesSearchResultsViewerTable.filesColLbl=Files", "CommonFilesSearchResultsViewerTable.instancesColLbl=Instances", @@ -63,18 +67,23 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { "CommonFilesSearchResultsViewerTable.tagsColLbl1=Tags" }) @Override - protected void setColumnWidths(){ + protected void setColumnWidths() { TableColumnModel model = this.getColumnModel(); - + Enumeration columnsEnumerator = model.getColumns(); - while(columnsEnumerator.hasMoreElements()){ - + while (columnsEnumerator.hasMoreElements()) { + TableColumn column = columnsEnumerator.nextElement(); - + final String headerValue = column.getHeaderValue().toString(); - final Integer get = COLUMN_WIDTHS.get(headerValue); - - column.setPreferredWidth(get); + + try { + final Integer get = COLUMN_WIDTHS.get(headerValue); + + column.setPreferredWidth(get); + } catch (NullPointerException e) { + LOGGER.log(Level.WARNING, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); + } } } } From f878ca5cd287737b4dc4062c604823645243b98b Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 10:30:24 -0400 Subject: [PATCH 18/96] 3978 add checks for confidence and size to Unicode extractor --- .../keywordsearch/UnicodeTextExtractor.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java index a2189b8f4c..d0159a3f6c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java @@ -29,10 +29,18 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream; /** - * Extract text from unicode files + * Extract text from unicode files */ final class UnicodeTextExtractor extends ContentTextExtractor { + //Set an upper limit on the amount of data in a single file to index + static final private int MAX_DATA_SIZE_BYTES = 100000000; + //Set a Minimum confidence value to reject matches that may not have a valid text encoding + //Values of valid text encodings were generally 100, xml code sometimes had a value around 50, + //and pictures and other files with a .txt extention were showing up with a value of 5 or less in limited testing. + //This limited information was used to select the current value as one that would filter out clearly non-text + //files while hopefully working on all files with a valid text encoding + static final private int MIN_MATCH_CONFIDENCE = 20; static final private Logger logger = Logger.getLogger(UnicodeTextExtractor.class.getName()); @Override @@ -49,11 +57,19 @@ final class UnicodeTextExtractor extends ContentTextExtractor { public Reader getReader(Content source) throws TextExtractorException { CharsetDetector detector = new CharsetDetector(); ReadContentInputStream stream = new ReadContentInputStream(source); - byte[] byteData = new byte[(int) source.getSize()]; + int size = (int) source.getSize(); + if (size > MAX_DATA_SIZE_BYTES) { + size = MAX_DATA_SIZE_BYTES; + logger.log(Level.WARNING, "Text file size exceeded 100 mb, ony the first 100 mb has been indexed"); + } + byte[] byteData = new byte[size]; try { - stream.read(byteData, 0, (int) source.getSize()); + stream.read(byteData, 0, size); detector.setText(byteData); CharsetMatch match = detector.detect(); + if (match.getConfidence() < MIN_MATCH_CONFIDENCE) { + throw new TextExtractorException("Text does not match any character set with a high enough confidence for UnicodeTextExtractor"); + } try { return new StringReader(match.getString()); From 7b6ce83acc6cc81027db7f7f96f93a423a8e1a76 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 11:10:50 -0600 Subject: [PATCH 19/96] removed some broken and uneeded code --- .../datamodel/CommonFileChildNodeLoading.java | 95 ------------------- .../datamodel/DisplayableItemNodeVisitor.java | 7 -- 2 files changed, 102 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java b/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java deleted file mode 100644 index 0c83a12f61..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * - * Autopsy Forensic Browser - * - * Copyright 2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datamodel; - -import java.util.LinkedHashMap; -import java.util.Map; -import org.openide.nodes.Children; -import org.openide.nodes.Sheet; -import org.openide.util.NbBundle; - -/** - * A dummy node used by the child factory to display while children are being - * generated - */ -public class CommonFileChildNodeLoading extends DisplayableItemNode { - - public CommonFileChildNodeLoading(Children children) { - super(children); - } - - @Override - public T accept(DisplayableItemNodeVisitor visitor) { - return visitor.visit(this); - } - - @Override - public boolean isLeafTypeNode() { - return true; - } - - @Override - public String getItemType() { - return getClass().getName(); - } - - @Override - protected Sheet createSheet() { - Sheet sheet = new Sheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - - Map map = new LinkedHashMap<>(); - map.put(CommonFileChildLoadingPropertyType.File.toString(), "Loading..."); - - final String NO_DESCR = Bundle.AbstractFsContentNode_noDesc_text(); - for (CommonFileChildLoadingPropertyType propType : CommonFileChildLoadingPropertyType.values()) { - final String propString = propType.toString(); - sheetSet.put(new NodeProperty<>(propString, propString, NO_DESCR, map.get(propString))); - } - - return sheet; - } - - /** - * Represents the sole column for the 'dummy' loading node. - */ - @NbBundle.Messages({ - "CommonFileChildLoadingPropertyType.fileColLbl=File" - }) - public enum CommonFileChildLoadingPropertyType { - - File(Bundle.CommonFileChildLoadingPropertyType_fileColLbl()); - - final private String displayString; - - private CommonFileChildLoadingPropertyType(String displayString) { - this.displayString = displayString; - } - - @Override - public String toString() { - return displayString; - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java b/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java index 0e9bd1fcba..9cd13a2f4a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java @@ -120,8 +120,6 @@ public interface DisplayableItemNodeVisitor { T visit(CommonFilesNode cfn); T visit(FileInstanceNode fin); - - T visit(CommonFileChildNodeLoading cfcnl); T visit(InstanceCountNode icn); @@ -213,11 +211,6 @@ public interface DisplayableItemNodeVisitor { return defaultVisit(icn); } - @Override - public T visit(CommonFileChildNodeLoading cfcnl) { - return defaultVisit(cfcnl); - } - @Override public T visit(DirectoryNode dn) { return defaultVisit(dn); From 20f7993114d9f013faa077c73eb5799b7fb5f87a Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 13:34:02 -0400 Subject: [PATCH 20/96] 4003 preserve existing comment when replacing tag --- .../autopsy/actions/ReplaceBlackboardArtifactTagAction.java | 6 +++--- .../sleuthkit/autopsy/actions/ReplaceContentTagAction.java | 4 ++-- .../src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java index 17fbb40e1b..a2895b4d55 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java @@ -64,14 +64,14 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction() { @Override @@ -91,7 +91,7 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction "# {1} - content obj id", "ReplaceContentTagAction.replaceTag.alert=Unable to replace tag {0} for {1}."}) @Override - protected void replaceTag(ContentTag oldTag, TagName newTagName, String comment) { + protected void replaceTag(ContentTag oldTag, TagName newTagName, String newComment) { new SwingWorker() { @Override @@ -84,7 +84,7 @@ public final class ReplaceContentTagAction extends ReplaceTagAction logger.log(Level.INFO, "Replacing tag {0} with tag {1} for artifact {2}", new Object[]{oldTag.getName().getDisplayName(), newTagName.getDisplayName(), oldTag.getContent().getName()}); //NON-NLS tagsManager.deleteContentTag(oldTag); - tagsManager.addContentTag(oldTag.getContent(), newTagName, comment); + tagsManager.addContentTag(oldTag.getContent(), newTagName, newComment); } catch (TskCoreException tskCoreException) { logger.log(Level.SEVERE, "Error replacing artifact tag", tskCoreException); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java index 03d48f4012..79ca4f4739 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java @@ -141,7 +141,7 @@ abstract class ReplaceTagAction extends AbstractAction implements // Add action to replace the tag tagNameItem.addActionListener((ActionEvent event) -> { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, entry.getValue(), ""); + replaceTag(oldtag, entry.getValue(), oldtag.getComment()); }); }); @@ -178,7 +178,7 @@ abstract class ReplaceTagAction extends AbstractAction implements TagName newTagName = GetTagNameDialog.doDialog(); if (null != newTagName) { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, newTagName, ""); + replaceTag(oldtag, newTagName, oldtag.getComment()); }); } }); From 6e998d137be05939f8f9f4b2457206b730d18c5c Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 11:38:20 -0600 Subject: [PATCH 21/96] removed debugging code --- .../CommonFilesSearchResultsViewerTable.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index fecb676f4b..e6221c5b50 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -77,13 +77,9 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final String headerValue = column.getHeaderValue().toString(); - try { - final Integer get = COLUMN_WIDTHS.get(headerValue); + final Integer get = COLUMN_WIDTHS.get(headerValue); - column.setPreferredWidth(get); - } catch (NullPointerException e) { - LOGGER.log(Level.WARNING, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); - } + column.setPreferredWidth(get); } } } From 0d8c299935d8ce6b7260e44396b9eb21d64af3c5 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 11:51:26 -0600 Subject: [PATCH 22/96] removed useless code --- .../commonfilesearch/CommonFilesSearchResultsViewerTable.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index e6221c5b50..db03d08dc6 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -23,8 +23,6 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; @@ -42,8 +40,6 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; - private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); - static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); From f1ea178b00c2f7c872368a125f0d66af5f15b089 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 12:08:19 -0600 Subject: [PATCH 23/96] handle error conditions gracefully --- .../CommonFilesSearchResultsViewerTable.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index db03d08dc6..9da6a61a21 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -40,6 +40,8 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; + private static final int DEFAULT_WIDTH = 100; + static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); @@ -73,9 +75,16 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final String headerValue = column.getHeaderValue().toString(); - final Integer get = COLUMN_WIDTHS.get(headerValue); + final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); - column.setPreferredWidth(get); + if (defaultWidth == null) { + //this should never happen, but if it did, it is because + // we've added a node to the tree that has columns which are + // not defined in the mapping above + column.setPreferredWidth(DEFAULT_WIDTH); + } else { + column.setPreferredWidth(defaultWidth); + } } } } From 9a11e33de055f12b167686ede26e687f358e3495 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 12:27:35 -0600 Subject: [PATCH 24/96] added exception logging back --- .../CommonFilesSearchResultsViewerTable.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index 9da6a61a21..56287dfe32 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; @@ -40,7 +42,7 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; - private static final int DEFAULT_WIDTH = 100; + private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); static { Map map = new HashMap<>(); @@ -76,14 +78,11 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final String headerValue = column.getHeaderValue().toString(); final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); - - if (defaultWidth == null) { - //this should never happen, but if it did, it is because - // we've added a node to the tree that has columns which are - // not defined in the mapping above - column.setPreferredWidth(DEFAULT_WIDTH); - } else { + + try { column.setPreferredWidth(defaultWidth); + } catch (NullPointerException e) { + LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); } } } From 22af40ab95359d1aee68927886ab614b83d5ed42 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 12:41:13 -0600 Subject: [PATCH 25/96] logging added back --- .../commonfilesearch/CommonFilesSearchResultsViewerTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index 56287dfe32..c005799782 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -81,7 +81,7 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { try { column.setPreferredWidth(defaultWidth); - } catch (NullPointerException e) { + } catch (Exception e) { LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); } } From 1c0643ac3e843b21f801ffe2ad3bf79b9ce93cb2 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 13:15:23 -0600 Subject: [PATCH 26/96] removed exception handling in favor of null checking --- .../CommonFilesSearchResultsViewerTable.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index c005799782..cacbdd83f9 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -43,6 +43,8 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final long serialVersionUID = 1L; private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); + + private static final int DEFAULT_WIDTH = 100; static { Map map = new HashMap<>(); @@ -79,10 +81,11 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); - try { + if(defaultWidth == null){ + column.setPreferredWidth(DEFAULT_WIDTH); + LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue)); + } else { column.setPreferredWidth(defaultWidth); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); } } } From 3b3e0b1a220954b4663de4890d31db53d6b9f5f0 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 15:26:49 -0400 Subject: [PATCH 27/96] 3978 simplify Unicode text extractor code per recommendations from review --- .../keywordsearch/UnicodeTextExtractor.java | 32 ++++++------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java index d0159a3f6c..cbd84c1fe7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.keywordsearch; import java.io.IOException; import java.io.Reader; -import java.io.StringReader; import java.util.logging.Level; import org.apache.tika.parser.txt.CharsetDetector; import org.apache.tika.parser.txt.CharsetMatch; @@ -33,8 +32,6 @@ import org.sleuthkit.datamodel.ReadContentInputStream; */ final class UnicodeTextExtractor extends ContentTextExtractor { - //Set an upper limit on the amount of data in a single file to index - static final private int MAX_DATA_SIZE_BYTES = 100000000; //Set a Minimum confidence value to reject matches that may not have a valid text encoding //Values of valid text encodings were generally 100, xml code sometimes had a value around 50, //and pictures and other files with a .txt extention were showing up with a value of 5 or less in limited testing. @@ -57,28 +54,17 @@ final class UnicodeTextExtractor extends ContentTextExtractor { public Reader getReader(Content source) throws TextExtractorException { CharsetDetector detector = new CharsetDetector(); ReadContentInputStream stream = new ReadContentInputStream(source); - int size = (int) source.getSize(); - if (size > MAX_DATA_SIZE_BYTES) { - size = MAX_DATA_SIZE_BYTES; - logger.log(Level.WARNING, "Text file size exceeded 100 mb, ony the first 100 mb has been indexed"); - } - byte[] byteData = new byte[size]; try { - stream.read(byteData, 0, size); - detector.setText(byteData); - CharsetMatch match = detector.detect(); - if (match.getConfidence() < MIN_MATCH_CONFIDENCE) { - throw new TextExtractorException("Text does not match any character set with a high enough confidence for UnicodeTextExtractor"); - } - try { - return new StringReader(match.getString()); - - } catch (IOException ex) { - throw new TextExtractorException("Unable to get string from detected text in UnicodeTextExtractor", ex); - } - } catch (ReadContentInputStream.ReadContentInputStreamException ex) { - throw new TextExtractorException("Unable to read text stream in UnicodeTextExtractor", ex); + detector.setText(stream); + } catch (IOException ex) { + throw new TextExtractorException("Unable to get string from detected text in UnicodeTextExtractor", ex); } + CharsetMatch match = detector.detect(); + if (match.getConfidence() < MIN_MATCH_CONFIDENCE) { + throw new TextExtractorException("Text does not match any character set with a high enough confidence for UnicodeTextExtractor"); + } + + return match.getReader(); } @Override From 46854d4f7a70080ab1126f3544240aadbd8562a8 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 17:59:42 -0400 Subject: [PATCH 28/96] 3978 rename UnicodeTextExtractor to TextFileExtractor to better reflect its purpose --- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 8 ++++---- .../{UnicodeTextExtractor.java => TextFileExtractor.java} | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) rename KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/{UnicodeTextExtractor.java => TextFileExtractor.java} (93%) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 3ed4f76b38..a8abc03b83 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -91,7 +91,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private boolean startedSearching = false; private List textExtractors; private StringsTextExtractor stringExtractor; - private UnicodeTextExtractor unicodeExtractor; + private TextFileExtractor txtFileExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private long jobId; @@ -244,7 +244,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); - unicodeExtractor = new UnicodeTextExtractor(); + txtFileExtractor = new TextFileExtractor(); textExtractors = new ArrayList<>(); //order matters, more specific extractors first @@ -345,7 +345,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { textExtractors.clear(); textExtractors = null; stringExtractor = null; - unicodeExtractor = null; + txtFileExtractor = null; initialized = false; } @@ -572,7 +572,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { if ((wasTextAdded == false) && (aFile.getNameExtension().equalsIgnoreCase("txt"))) { try { - if (Ingester.getDefault().indexText(unicodeExtractor, aFile, context)) { + if (Ingester.getDefault().indexText(txtFileExtractor, aFile, context)) { putIngestStatus(jobId, aFile.getId(), IngestStatus.TEXT_INGESTED); wasTextAdded = true; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java similarity index 93% rename from KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java rename to KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java index cbd84c1fe7..bc11515e96 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java @@ -28,9 +28,9 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream; /** - * Extract text from unicode files + * Extract text from .txt files */ -final class UnicodeTextExtractor extends ContentTextExtractor { +final class TextFileExtractor extends ContentTextExtractor { //Set a Minimum confidence value to reject matches that may not have a valid text encoding //Values of valid text encodings were generally 100, xml code sometimes had a value around 50, @@ -38,7 +38,7 @@ final class UnicodeTextExtractor extends ContentTextExtractor { //This limited information was used to select the current value as one that would filter out clearly non-text //files while hopefully working on all files with a valid text encoding static final private int MIN_MATCH_CONFIDENCE = 20; - static final private Logger logger = Logger.getLogger(UnicodeTextExtractor.class.getName()); + static final private Logger logger = Logger.getLogger(TextFileExtractor.class.getName()); @Override boolean isContentTypeSpecific() { From 4d95be231b599689cca01fd0f41bb96d2b39c3fd Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 2 Jul 2018 09:12:16 -0400 Subject: [PATCH 29/96] Don't try to add new data sources to the central repo twice. Don't throw an exception when attempting to add an already initialized data source --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 3 ++- .../centralrepository/datamodel/CorrelationDataSource.java | 1 + .../centralrepository/eventlisteners/CaseEventListener.java | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 767e86ebcc..3066340c55 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -432,7 +432,8 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamDataSource.getCaseID() == -1) { throw new EamDbException("Case ID is -1"); } else if (eamDataSource.getID() != -1) { - throw new EamDbException("Database ID is already set in object"); + // This data source is already in the central repo + return; } Connection conn = connect(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java index 8cf07a42e9..84dce834a1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java @@ -67,6 +67,7 @@ public class CorrelationDataSource implements Serializable { /** * Create a CorrelationDataSource object from a TSK Content object. + * This will add it to the central repository. * * @param correlationCase the current CorrelationCase used for ensuring * uniqueness of DataSource diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 55958287d5..a3d49d6ba3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -450,10 +450,10 @@ final class CaseEventListener implements PropertyChangeListener { correlationCase = dbManager.newCase(openCase); } if (null == dbManager.getDataSource(correlationCase, deviceId)) { - dbManager.newDataSource(CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource)); + CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS + LOGGER.log(Level.SEVERE, "Error adding new data source to the central repository", ex); //NON-NLS } catch (TskCoreException | TskDataException ex) { LOGGER.log(Level.SEVERE, "Error getting data source from DATA_SOURCE_ADDED event content.", ex); //NON-NLS } From 2d279350693688c45d99b80107f3c488c208aa6d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 2 Jul 2018 13:07:29 -0400 Subject: [PATCH 30/96] 4006 do not escape cell contents when writing thumbnails --- .../sleuthkit/autopsy/report/ReportHTML.java | 102 ++++++++++-------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index b69d84b629..50dfe318c3 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -556,17 +556,29 @@ class ReportHTML implements TableReportModule { } /** - * Add a row to the current table. + * Add a row to the current table, escaping the text to be contained in the + * row. * * @param row values for each cell in the row */ @Override public void addRow(List row) { + addRow(row, true); + } + + /** + * Add a row to the current table. + * + * @param row values for each cell in the row + * @param escapeText whether or not the text of the row should be escaped, + * true for escaped, false for not escaped + */ + private void addRow(List row, boolean escapeText) { StringBuilder builder = new StringBuilder(); builder.append("\t\n"); //NON-NLS for (String cell : row) { - String escapeHTMLCell = EscapeUtil.escapeHtml(cell); - builder.append("\t\t").append(escapeHTMLCell).append("\n"); //NON-NLS + String cellText = escapeText ? EscapeUtil.escapeHtml(cell) : cell; + builder.append("\t\t").append(cellText).append("\n"); //NON-NLS } builder.append("\t\n"); //NON-NLS rowCount++; @@ -593,7 +605,7 @@ class ReportHTML implements TableReportModule { public void addRowWithTaggedContentHyperlink(List row, ContentTag contentTag) { Content content = contentTag.getContent(); if (content instanceof AbstractFile == false) { - addRow(row); + addRow(row, true); return; } AbstractFile file = (AbstractFile) content; @@ -647,7 +659,7 @@ class ReportHTML implements TableReportModule { int pages = 1; for (Content content : images) { if (currentRow.size() == THUMBNAIL_COLUMNS) { - addRow(currentRow); + addRow(currentRow, false); currentRow.clear(); } @@ -727,7 +739,7 @@ class ReportHTML implements TableReportModule { // Finish out the row. currentRow.add(""); } - addRow(currentRow); + addRow(currentRow, false); } // manually set rowCount to be the total number of images. @@ -1094,8 +1106,8 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS summary.append(writeSummaryCaseDetails()); summary.append(writeSummaryImageInfo()); - summary.append(writeSummarySoftwareInfo(skCase,ingestJobs)); - summary.append(writeSummaryIngestHistoryInfo(skCase,ingestJobs)); + summary.append(writeSummarySoftwareInfo(skCase, ingestJobs)); + summary.append(writeSummaryIngestHistoryInfo(skCase, ingestJobs)); if (generatorLogoSet) { summary.append("
\n"); //NON-NLS summary.append("\n"); //NON-NLS @@ -1126,14 +1138,13 @@ class ReportHTML implements TableReportModule { } } } - + /** * Write the case details section of the summary for this report. - * + * * @return StringBuilder updated html report with case details */ - - private StringBuilder writeSummaryCaseDetails(){ + private StringBuilder writeSummaryCaseDetails() { StringBuilder summary = new StringBuilder(); String caseName = currentCase.getDisplayName(); String caseNumber = currentCase.getNumber(); @@ -1146,40 +1157,39 @@ class ReportHTML implements TableReportModule { imagecount = 0; } summary.append("
\n"); //NON-NLS - if (agencyLogoSet) { - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - } - final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS - .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS - .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS - .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS - .append(!examiner.isEmpty() ? examiner : NbBundle - .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) - .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS - .append("").append(imagecount).append("
\n"); //NON-NLS + if (agencyLogoSet) { + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - return summary; + } + final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS + .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS + .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS + .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS + .append(!examiner.isEmpty() ? examiner : NbBundle + .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) + .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS + .append("").append(imagecount).append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + return summary; } - + /** * Write the Image Information section of the summary for this report. - * + * * @return StringBuilder updated html report with Image Information */ - private StringBuilder writeSummaryImageInfo() { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.imageInfoHeading")); @@ -1208,13 +1218,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the software information section of the summary for this report. - * + * * @return StringBuilder updated html report with software information */ - private StringBuilder writeSummarySoftwareInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.softwareInfoHeading")); @@ -1244,13 +1253,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the Ingest History section of the summary for this report. - * + * * @return StringBuilder updated html report with ingest history */ - private StringBuilder writeSummaryIngestHistoryInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); try { @@ -1304,4 +1312,4 @@ class ReportHTML implements TableReportModule { + thumbFile.getName(); } -} \ No newline at end of file +} From f4448359029a8d6d16f235eb3f6378fbb53aa13a Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 08:51:14 -0400 Subject: [PATCH 31/96] Add max length for central repo value field --- .../AddEditCentralRepoCommentAction.java | 2 +- .../datamodel/AbstractSqlEamDb.java | 35 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index dfeb4fbdcd..654ee7161e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -105,7 +105,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { dbManager.updateAttributeInstanceComment(correlationAttribute); } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error adding comment", ex); } } return centralRepoCommentDialog.getComment(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 767e86ebcc..3d4bd946ea 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -57,7 +57,10 @@ abstract class AbstractSqlEamDb implements EamDb { private int bulkArtifactsCount; protected int bulkArtifactsThreshold; private final Map> bulkArtifacts; - + + // Maximum length for the value column in the instance tables + static final int MAX_VALUE_LENGTH = 128; + // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. static final int DEFAULT_BULK_THRESHHOLD = 1000; @@ -550,6 +553,12 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationValue() == null) { throw new EamDbException("Correlation value is null"); } + + // Don't throw an exception here because the calling code may be looping over values. + if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { + logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); + return; + } Connection conn = connect(); @@ -984,18 +993,22 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("CorrelationAttributeInstance known status is null"); } - bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); - bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); - bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); - bulkPs.setString(4, eamArtifact.getCorrelationValue()); - bulkPs.setString(5, eamInstance.getFilePath()); - bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); - if ("".equals(eamInstance.getComment())) { - bulkPs.setNull(7, Types.INTEGER); + if (eamArtifact.getCorrelationValue().length() < MAX_VALUE_LENGTH) { + bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); + bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); + bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); + bulkPs.setString(4, eamArtifact.getCorrelationValue()); + bulkPs.setString(5, eamInstance.getFilePath()); + bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); + if ("".equals(eamInstance.getComment())) { + bulkPs.setNull(7, Types.INTEGER); + } else { + bulkPs.setString(7, eamInstance.getComment()); + } + bulkPs.addBatch(); } else { - bulkPs.setString(7, eamInstance.getComment()); + logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); } - bulkPs.addBatch(); } } } From ba6f406add181a7f29ee3a34cf1ef12225169717 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 13:04:14 -0400 Subject: [PATCH 32/96] Add dialog to allow user to switch to grouping view for large cases --- .../sleuthkit/autopsy/casemodule/Case.java | 11 ++ .../autopsy/directorytree/Bundle.properties | 5 + .../DirectoryTreeTopComponent.java | 82 ++++++++++ .../directorytree/GroupDataSourcesDialog.form | 101 ++++++++++++ .../directorytree/GroupDataSourcesDialog.java | 148 ++++++++++++++++++ 5 files changed, 347 insertions(+) create mode 100644 Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form create mode 100644 Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 44bf52503f..a9bbb39907 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -129,6 +129,7 @@ public class Case { private static final String EXPORT_FOLDER = "Export"; //NON-NLS private static final String LOG_FOLDER = "Log"; //NON-NLS private static final String REPORTS_FOLDER = "Reports"; //NON-NLS + private static final String CONFIG_FOLDER = "Config"; // NON-NLS private static final String TEMP_FOLDER = "Temp"; //NON-NLS private static final String MODULE_FOLDER = "ModuleOutput"; //NON-NLS private static final String CASE_ACTION_THREAD_NAME = "%s-case-action"; @@ -1350,6 +1351,16 @@ public class Case { public String getReportDirectory() { return getOrCreateSubdirectory(REPORTS_FOLDER); } + + /** + * Gets the full path to the config directory for this case, creating it if + * it does not exist. + * + * @return The config directory path. + */ + public String getConfigDirectory() { + return getOrCreateSubdirectory(CONFIG_FOLDER); + } /** * Gets the full path to the module output directory for this case, creating diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties index 0e7f3cf315..926c4b4626 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties @@ -120,3 +120,8 @@ AddExternalViewerRulePanel.exePathTextField.text= AddExternalViewerRulePanel.exePathLabel.text=Path of the program to use for files with this type or extension AddExternalViewerRulePanel.extRadioButton.text=Extension DirectoryTreeTopComponent.groupByDatasourceCheckBox.text=Group by Data Source +GroupDataSourcesDialog.dataSourceCountLabel.text=jLabel1 +GroupDataSourcesDialog.queryLabel.text=Would you like to group by data source for faster loading? +GroupDataSourcesDialog.yesButton.text=Yes +GroupDataSourcesDialog.noButton.text=No +GroupDataSourcesDialog.title=Group by Data Source? \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 7db201f00b..66bf153833 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -24,6 +24,11 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyVetoException; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -35,6 +40,7 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.prefs.PreferenceChangeEvent; import java.util.prefs.PreferenceChangeListener; +import java.util.Properties; import javax.swing.Action; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; @@ -62,6 +68,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; import org.sleuthkit.autopsy.corecomponents.TableFilterNode; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.datamodel.ArtifactNodeSelectionInfo; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; import org.sleuthkit.autopsy.datamodel.CreditCards; @@ -103,6 +110,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private static final Logger LOGGER = Logger.getLogger(DirectoryTreeTopComponent.class.getName()); private AutopsyTreeChildrenFactory autopsyTreeChildrenFactory; private Children autopsyTreeChildren; + private final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source + private final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; + private final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS /** * the constructor @@ -369,7 +379,51 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat public int getPersistenceType() { return TopComponent.PERSISTENCE_NEVER; } + + /** + * Ask the user if they want to group by data source when opening a large case. + * + * @param currentCase + * @param dataSourceCount + */ + private void promptForDataSourceGrouping(Case currentCase, int dataSourceCount) { + Path settingsFile = Paths.get(currentCase.getConfigDirectory(), SETTINGS_FILE); //NON-NLS + if (settingsFile.toFile().exists()) { + // Read the setting + try (InputStream inputStream = Files.newInputStream(settingsFile)) { + Properties props = new Properties(); + props.load(inputStream); + if (props.getProperty("groupByDataSource", "false").equals("true")) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error reading settings file", ex); + } + } else { + GroupDataSourcesDialog dialog = new GroupDataSourcesDialog(dataSourceCount); + dialog.display(); + if (dialog.groupByDataSourceSelected()) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + // Save the response + Properties props = new Properties(); + if(dialog.groupByDataSourceSelected()) { + props.setProperty("groupByDataSource", "true"); + } else { + props.setProperty("groupByDataSource", "false"); + } + + try (OutputStream fos = Files.newOutputStream(settingsFile)) { + props.store(fos, ""); //NON-NLS + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error writing settings file", ex); + } + } + } + /** * Called only when top component was closed on all workspaces before and * now is opened for the first time on some workspace. The intent is to @@ -377,6 +431,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * existing workspaces. Subclasses will usually perform initializing tasks * here. */ + @NbBundle.Messages({"# {0} - dataSourceCount", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) @Override public void componentOpened() { // change the cursor to "waiting cursor" for this operation @@ -392,6 +449,31 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat if (null == currentCase || currentCase.hasData() == false) { getTree().setRootVisible(false); // hide the root } else { + // If the case contains a lot of data sources, and they aren't already grouping + // by data source, give the user the option to do so before loading the tree. + if (RuntimeProperties.runningWithGUI()) { + long threshold = DEFAULT_DATASOURCE_GROUPING_THRESHOLD; + if (ModuleSettings.settingExists(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)) { + try { + threshold = Long.parseLong(ModuleSettings.getConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)); + } catch (NumberFormatException ex) { + LOGGER.log(Level.SEVERE, "Group data sources threshold is not a number", ex); + } + } else { + ModuleSettings.setConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME, String.valueOf(threshold)); + } + + try { + int dataSourceCount = currentCase.getDataSources().size(); + if (! UserPreferences.groupItemsInTreeByDatasource() && + dataSourceCount > threshold) { + promptForDataSourceGrouping(currentCase, dataSourceCount); + } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error loading data sources", ex); + } + } + // if there's at least one image, load the image and open the top componen autopsyTreeChildrenFactory = new AutopsyTreeChildrenFactory(); autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form new file mode 100644 index 0000000000..e31c97d4ba --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form @@ -0,0 +1,101 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java new file mode 100644 index 0000000000..f21c1efca9 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java @@ -0,0 +1,148 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.directorytree; + +import javax.swing.JFrame; +import org.openide.util.NbBundle; +import org.openide.windows.WindowManager; +import org.sleuthkit.autopsy.core.UserPreferences; + +/** + * + */ +@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives +final class GroupDataSourcesDialog extends javax.swing.JDialog { + + boolean shouldGroupByDataSource = false; + + /** + * Creates new form GroupDataSourcesDialog + */ + @NbBundle.Messages({"# {0} - dataSourceCount", + "GroupDataSourcesDialog.groupDataSources.text=This case contains {0} data sources."}) + GroupDataSourcesDialog(int dataSourceCount) { + super((JFrame) WindowManager.getDefault().getMainWindow()); + initComponents(); + dataSourceCountLabel.setText(Bundle.GroupDataSourcesDialog_groupDataSources_text(dataSourceCount)); + } + + /** + * Display the dialog. + */ + void display() { + setModal(true); + setSize(getPreferredSize()); + setLocationRelativeTo(this.getParent()); + setAlwaysOnTop(false); + pack(); + setVisible(true); + } + + boolean groupByDataSourceSelected() { + return shouldGroupByDataSource; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + dataSourceCountLabel = new javax.swing.JLabel(); + queryLabel = new javax.swing.JLabel(); + yesButton = new javax.swing.JButton(); + noButton = new javax.swing.JButton(); + + setDefaultCloseOperation(javax.swing.WindowConstants.DISPOSE_ON_CLOSE); + setTitle(org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.title")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(dataSourceCountLabel, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.dataSourceCountLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(queryLabel, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.queryLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(yesButton, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.yesButton.text")); // NOI18N + yesButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + yesButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(noButton, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.noButton.text")); // NOI18N + noButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + noButtonActionPerformed(evt); + } + }); + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); + getContentPane().setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(queryLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createSequentialGroup() + .addComponent(dataSourceCountLabel) + .addGap(0, 0, Short.MAX_VALUE)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addGap(0, 0, Short.MAX_VALUE) + .addComponent(yesButton, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(noButton, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addContainerGap()) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(dataSourceCountLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(queryLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(yesButton) + .addComponent(noButton)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + ); + + pack(); + }// //GEN-END:initComponents + + private void yesButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_yesButtonActionPerformed + shouldGroupByDataSource = true; + dispose(); + }//GEN-LAST:event_yesButtonActionPerformed + + private void noButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_noButtonActionPerformed + shouldGroupByDataSource = false; + dispose(); + }//GEN-LAST:event_noButtonActionPerformed + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JLabel dataSourceCountLabel; + private javax.swing.JButton noButton; + private javax.swing.JLabel queryLabel; + private javax.swing.JButton yesButton; + // End of variables declaration//GEN-END:variables +} From d9578cdfb8270d19a303042dfea75ca6352482c2 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 13:09:03 -0400 Subject: [PATCH 33/96] Throw an exception --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 5 +---- .../eventlisteners/IngestEventsListener.java | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 3d4bd946ea..628e6a962a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -553,11 +553,8 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationValue() == null) { throw new EamDbException("Correlation value is null"); } - - // Don't throw an exception here because the calling code may be looping over values. if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { - logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); - return; + throw new EamDbException("Artifact value too long for central repository: " + eamArtifact.getCorrelationValue()); } Connection conn = connect(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 46ebe75541..e3cfad9fe8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -276,12 +276,12 @@ public class IngestEventsListener { } } if (FALSE == eamArtifacts.isEmpty()) { - try { - for (CorrelationAttribute eamArtifact : eamArtifacts) { + for (CorrelationAttribute eamArtifact : eamArtifacts) { + try { dbManager.addArtifact(eamArtifact); + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS } } // DATA_ADDED } From 660889d33b4c495342c22100b329404b3f241bbe Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 13:26:00 -0400 Subject: [PATCH 34/96] Codacy --- .../autopsy/directorytree/DirectoryTreeTopComponent.java | 6 +++--- .../autopsy/directorytree/GroupDataSourcesDialog.java | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 66bf153833..97b99103bd 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -110,9 +110,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private static final Logger LOGGER = Logger.getLogger(DirectoryTreeTopComponent.class.getName()); private AutopsyTreeChildrenFactory autopsyTreeChildrenFactory; private Children autopsyTreeChildren; - private final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source - private final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; - private final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS + private static final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source + private static final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; + private static final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS /** * the constructor diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java index f21c1efca9..7641655d57 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.directorytree; import javax.swing.JFrame; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.core.UserPreferences; /** * From 842baa2ec3c6a4d3052ae2d84a0dcd3c27e4f12d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 5 Jul 2018 12:44:25 -0400 Subject: [PATCH 35/96] Disable data source search when only one available. --- .../autopsy/filesearch/DataSourcePanel.java | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java index 96d7973d8a..64fec82a92 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java @@ -54,31 +54,42 @@ public class DataSourcePanel extends javax.swing.JPanel { */ public DataSourcePanel() { initComponents(); - this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { - firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); - }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } + if (this.dataSourceList.getModel().getSize() > 1) { + this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + }); + this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - @Override - public void mouseMoved(MouseEvent evt) { - if (evt.getSource() instanceof JList) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(toolTipList.get(index)); + @Override + public void mouseDragged(MouseEvent evt) { + //Unused by now + } + + @Override + public void mouseMoved(MouseEvent evt) { + if (evt.getSource() instanceof JList) { + JList dsList = (JList) evt.getSource(); + int index = dsList.locationToIndex(evt.getPoint()); + if (index > -1) { + dsList.setToolTipText(toolTipList.get(index)); + } } } - } - }); + }); + } else { + /* + * Disable data source filtering since there aren't multiple data + * sources to choose from. + */ + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceList.setEnabled(false); + } } /** - * Get dataSourceMap with object id and data source display name. Add the data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. Add the + * data source full name to toolTipList * * @return The list of data source name */ @@ -93,7 +104,7 @@ public class DataSourcePanel extends javax.swing.JPanel { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); - dataSourceMap.put(ds.getId(), displayName); + dataSourceMap.put(ds.getId(), displayName); toolTipList.add(dsName); dsList.add(displayName); } @@ -107,7 +118,8 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Get a set of data source object ids that are selected. - * @return A set of selected object ids. + * + * @return A set of selected object ids. */ Set getDataSourcesSelected() { Set dataSourceObjIdSet = new HashSet<>(); @@ -124,6 +136,7 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Is dataSourceCheckBox selected + * * @return true if the dataSoureCheckBox is selected */ boolean isSelected() { @@ -131,7 +144,8 @@ public class DataSourcePanel extends javax.swing.JPanel { } /** - * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is checked. + * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is + * checked. */ final void setComponentsEnabled() { boolean enabled = this.isSelected(); From 083b8d0bfd94f46e765a01ee304fce75e6486f79 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 5 Jul 2018 15:56:03 -0400 Subject: [PATCH 36/96] 3996 improve opencv logging and prevent any opencv exceptions from escaping process method --- .../ObjectDetectectionFileIngestModule.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index fe86fb9f0b..ca0dd1cf89 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -106,10 +106,22 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter try { imageInMemory = IOUtils.toByteArray(inputStream); } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to perform object detection on " + file.getName(), ex); + logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getName(), ex); return IngestModule.ProcessResult.ERROR; } - Mat originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + Mat originalImage; + try { + originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + } catch (CvException ex) { + //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getName(), ex); //NON-NLS + return IngestModule.ProcessResult.ERROR; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getName(), unexpectedException); + return IngestModule.ProcessResult.ERROR; + } + MatOfRect detectionRectangles = new MatOfRect(); //the rectangles which reprent the coordinates on the image for where objects were detected for (String classifierKey : classifiers.keySet()) { //apply each classifier to the file @@ -119,6 +131,10 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName())); //NON-NLS continue; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); + continue; } if (!detectionRectangles.empty()) { From f2e0660829ead6b51b617fcd4d625867ec8115a2 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 5 Jul 2018 16:05:55 -0400 Subject: [PATCH 37/96] 3996 add parent path to logging messages for file for easier finding --- .../objectdetection/ObjectDetectectionFileIngestModule.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index ca0dd1cf89..fd0626041d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -114,11 +114,11 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); } catch (CvException ex) { //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us - logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getName(), ex); //NON-NLS + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName(), ex); //NON-NLS return IngestModule.ProcessResult.ERROR; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getName(), unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName(), unexpectedException); return IngestModule.ProcessResult.ERROR; } @@ -133,7 +133,7 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter continue; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); continue; } From 3b8b38da6f63d364cc6b268a0425e26f96fe4a65 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 5 Jul 2018 16:12:46 -0400 Subject: [PATCH 38/96] 3996 add object id to log messages for OpenCV related object detection exceptions --- .../ObjectDetectectionFileIngestModule.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index fd0626041d..3e58aac8c1 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -106,7 +106,7 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter try { imageInMemory = IOUtils.toByteArray(inputStream); } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getName(), ex); + logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); return IngestModule.ProcessResult.ERROR; } Mat originalImage; @@ -114,11 +114,11 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); } catch (CvException ex) { //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us - logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName(), ex); //NON-NLS + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); //NON-NLS return IngestModule.ProcessResult.ERROR; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName(), unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), unexpectedException); return IngestModule.ProcessResult.ERROR; } @@ -129,11 +129,11 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter classifiers.get(classifierKey).detectMultiScale(originalImage, detectionRectangles); } catch (CvException ignored) { //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable - logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName())); //NON-NLS + logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName() + " with object id of " + file.getId())); //NON-NLS continue; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " with object id of " + file.getId() +" while trying to apply classifier " + classifierKey, unexpectedException); continue; } From 5e2f51235871ad4553357c3dda31481d036e38cd Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 6 Jul 2018 11:08:43 -0400 Subject: [PATCH 39/96] 4017 improve logging in regards to when a correlations artifacts value is too long --- .../datamodel/AbstractSqlEamDb.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 9cba07f837..78937a221f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -57,10 +57,10 @@ abstract class AbstractSqlEamDb implements EamDb { private int bulkArtifactsCount; protected int bulkArtifactsThreshold; private final Map> bulkArtifacts; - + // Maximum length for the value column in the instance tables static final int MAX_VALUE_LENGTH = 128; - + // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. static final int DEFAULT_BULK_THRESHHOLD = 1000; @@ -555,7 +555,11 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation value is null"); } if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { - throw new EamDbException("Artifact value too long for central repository: " + eamArtifact.getCorrelationValue()); + throw new EamDbException("Artifact value too long for central repository." + + "\nCorrelationArtifact ID: " + eamArtifact.getID() + + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); + } Connection conn = connect(); @@ -982,7 +986,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (!eamArtifact.getCorrelationValue().isEmpty()) { if (eamInstance.getCorrelationCase() == null) { - throw new EamDbException("CorrelationAttributeInstance case is null"); + throw new EamDbException("CorrelationAttributeInstance case is null for :"); } if (eamInstance.getCorrelationDataSource() == null) { throw new EamDbException("CorrelationAttributeInstance data source is null"); @@ -1005,7 +1009,14 @@ abstract class AbstractSqlEamDb implements EamDb { } bulkPs.addBatch(); } else { - logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); + logger.log(Level.WARNING, ("Artifact value too long for central repository." + + "\nCorrelationArtifact ID: " + eamArtifact.getID() + + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) + + "\nEam Instance: " + + "\n\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() + + "\n\tFilePath: " + eamInstance.getFilePath()); } } } @@ -1751,11 +1762,13 @@ abstract class AbstractSqlEamDb implements EamDb { return 0 < badInstances; } + /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance + * * @throws EamDbException */ @Override From c819391112c7d54ec3bc0322603a08d9c2125892 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 6 Jul 2018 11:24:32 -0400 Subject: [PATCH 40/96] 3997 improve logging for recent activity module and cating of illegal arguement exception --- .../recentactivity/SearchEngineURLQueryAnalyzer.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index e5d93a91d4..fdaa47e7ed 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -237,8 +237,13 @@ class SearchEngineURLQueryAnalyzer extends Extract { try { //try to decode the url String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS return decoded; - } catch (UnsupportedEncodingException uee) { //if it fails, return the encoded string - logger.log(Level.FINE, "Error during URL decoding ", uee); //NON-NLS + } catch (UnsupportedEncodingException | IllegalArgumentException exception) { //if it fails, return the encoded string + logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:" + + "\nURL: " + url + + "\nUndecoded value: " + x + + "\nEngine name: " + eng.getEngineName() + + "\nEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS + return x; } } From f720d64f09a08c4b43b9b6ee7f752025bc9ce8b9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 6 Jul 2018 11:28:50 -0400 Subject: [PATCH 41/96] Removed tooltips. --- .../autopsy/filesearch/DataSourcePanel.java | 27 ++----------------- .../keywordsearch/AdHocSearchPanel.java | 14 +--------- .../DropdownListSearchPanel.java | 19 ------------- .../DropdownSingleTermSearchPanel.java | 19 ------------- 4 files changed, 3 insertions(+), 76 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java index 96d7973d8a..72877d5054 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.filesearch; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.io.File; import java.util.ArrayList; import java.util.Collections; @@ -29,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.event.ListSelectionEvent; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -47,7 +44,6 @@ public class DataSourcePanel extends javax.swing.JPanel { private static final Logger logger = Logger.getLogger(DataSourcePanel.class.getName()); private static final long serialVersionUID = 1L; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); /** * Creates new form DataSourcePanel @@ -57,28 +53,10 @@ public class DataSourcePanel extends javax.swing.JPanel { this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - if (evt.getSource() instanceof JList) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(toolTipList.get(index)); - } - } - } - }); } /** - * Get dataSourceMap with object id and data source display name. Add the data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ @@ -93,8 +71,7 @@ public class DataSourcePanel extends javax.swing.JPanel { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); - dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); + dataSourceMap.put(ds.getId(), displayName); dsList.add(displayName); } } catch (NoCurrentCaseException ex) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index c06394bf23..a8dbfb6268 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -43,7 +43,6 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { private final String keywordSearchErrorDialogHeader = org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.dialogErrorHeader"); protected int filesIndexed; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); private List dataSources = new ArrayList<>(); private final DefaultListModel dataSourceListModel = new DefaultListModel<>(); @@ -153,14 +152,12 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { */ synchronized List getDataSourceArray() { List dsList = new ArrayList<>(); - toolTipList.clear(); Collections.sort(this.dataSources, (DataSource ds1, DataSource ds2) -> ds1.getName().compareTo(ds2.getName())); for (DataSource ds : dataSources) { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); dsList.add(displayName); } return dsList; @@ -175,22 +172,13 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { } /** - * Get dataSourceMap with object id and data source display name. Add the - * data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ Map getDataSourceMap() { return dataSourceMap; } - - /** - * Get a list of tooltip text for data source - * @return A list of tool tips - */ - List getDataSourceToolTipList() { - return toolTipList; - } /** * Get a list of DataSourceListModel diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index e5c4191201..0c7566f52e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -23,8 +23,6 @@ import java.awt.Cursor; import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -34,7 +32,6 @@ import java.util.List; import java.util.Set; import java.util.logging.Level; import javax.swing.JCheckBox; -import javax.swing.JList; import javax.swing.JTable; import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; @@ -75,22 +72,6 @@ class DropdownListSearchPanel extends AdHocSearchPanel { dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } static synchronized DropdownListSearchPanel getDefault() { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java index ba872b0e4d..bdc80084c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java @@ -23,8 +23,6 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.FocusEvent; import java.awt.event.FocusListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -32,7 +30,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.JMenuItem; import javax.swing.event.ListSelectionEvent; import org.openide.util.NbBundle; @@ -88,22 +85,6 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList DsList = (JList) evt.getSource(); - int index = DsList.locationToIndex(evt.getPoint()); - if (index > -1) { - DsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } /** From 420a3fffe8681cd56518013fea6ee4e65b73bbda Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 6 Jul 2018 12:11:18 -0400 Subject: [PATCH 42/96] 3997 make new logger message SEVERE leave existing one as FINE --- .../datamodel/AbstractSqlEamDb.java | 32 +++++++++++++------ .../SearchEngineURLQueryAnalyzer.java | 26 +++++++++------ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 78937a221f..b405965b6b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -986,13 +986,25 @@ abstract class AbstractSqlEamDb implements EamDb { if (!eamArtifact.getCorrelationValue().isEmpty()) { if (eamInstance.getCorrelationCase() == null) { - throw new EamDbException("CorrelationAttributeInstance case is null for :"); + throw new EamDbException("CorrelationAttributeInstance case is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getCorrelationDataSource() == null) { - throw new EamDbException("CorrelationAttributeInstance data source is null"); + throw new EamDbException("CorrelationAttributeInstance data source is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getKnownStatus() == null) { - throw new EamDbException("CorrelationAttributeInstance known status is null"); + throw new EamDbException("CorrelationAttributeInstance known status is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue() + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID()); } if (eamArtifact.getCorrelationValue().length() < MAX_VALUE_LENGTH) { @@ -1010,13 +1022,13 @@ abstract class AbstractSqlEamDb implements EamDb { bulkPs.addBatch(); } else { logger.log(Level.WARNING, ("Artifact value too long for central repository." - + "\nCorrelationArtifact ID: " + eamArtifact.getID() - + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() - + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) - + "\nEam Instance: " - + "\n\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() - + "\n\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() - + "\n\tFilePath: " + eamInstance.getFilePath()); + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() + + "\n\t\tFilePath: " + eamInstance.getFilePath()); } } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index fdaa47e7ed..7bde8113a8 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2012-2014 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -237,13 +237,19 @@ class SearchEngineURLQueryAnalyzer extends Extract { try { //try to decode the url String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS return decoded; - } catch (UnsupportedEncodingException | IllegalArgumentException exception) { //if it fails, return the encoded string + } catch (UnsupportedEncodingException exception) { //if it fails, return the encoded string logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:" - + "\nURL: " + url - + "\nUndecoded value: " + x - + "\nEngine name: " + eng.getEngineName() - + "\nEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS - + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS + return x; + } catch (IllegalArgumentException exception) { //if it fails, return the encoded string + logger.log(Level.SEVERE, "Illegal arguement Exception during URL decoding, returning undecoded value:" + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS) return x; } } From 3539b6c70e700b5647559dbec1b40ff07fa8f378 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 6 Jul 2018 15:51:53 -0400 Subject: [PATCH 43/96] Renamed menu item; partial implementation of error handling. --- .../AddEditCentralRepoCommentAction.java | 59 ++++--------------- .../CentralRepoCommentDialog.java | 21 +++++-- ...CentralRepoContextMenuActionsProvider.java | 2 +- .../contentviewer/Bundle.properties | 2 +- .../DataContentViewerOtherCases.java | 5 +- 5 files changed, 31 insertions(+), 58 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index 654ee7161e..90317ab747 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -22,36 +22,32 @@ import java.awt.event.ActionEvent; import java.util.logging.Level; import javax.swing.AbstractAction; import org.openide.util.NbBundle.Messages; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; /** * An AbstractAction to manage adding and modifying a Central Repository file * instance comment. */ +@Messages({"AddEditCentralRepoCommentAction.menuItemText.addEditCentralRepoComment=Add/Edit Central Repository Comment"}) public final class AddEditCentralRepoCommentAction extends AbstractAction { private static final Logger logger = Logger.getLogger(AddEditCentralRepoCommentAction.class.getName()); private boolean addToDatabase; private CorrelationAttribute correlationAttribute; - String title; /** * Private constructor to create an instance given a CorrelationAttribute. * * @param correlationAttribute The correlation attribute to modify. - * @param title The text for the menu item. */ - private AddEditCentralRepoCommentAction(CorrelationAttribute correlationAttribute, String title) { - super(title); - this.title = title; + public AddEditCentralRepoCommentAction(CorrelationAttribute correlationAttribute) { + super(Bundle.AddEditCentralRepoCommentAction_menuItemText_addEditCentralRepoComment()); this.correlationAttribute = correlationAttribute; } @@ -60,12 +56,9 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { * * @param file The file from which a correlation attribute to modify is * derived. - * @param title The text for the menu item. */ - private AddEditCentralRepoCommentAction(AbstractFile file, String title) { - - super(title); - this.title = title; + public AddEditCentralRepoCommentAction(AbstractFile file) { + super(Bundle.AddEditCentralRepoCommentAction_menuItemText_addEditCentralRepoComment()); correlationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(file); if (correlationAttribute == null) { addToDatabase = true; @@ -90,7 +83,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { * @return the current comment for this instance */ public String addEditCentralRepoComment() { - CentralRepoCommentDialog centralRepoCommentDialog = new CentralRepoCommentDialog(correlationAttribute, title); + CentralRepoCommentDialog centralRepoCommentDialog = new CentralRepoCommentDialog(correlationAttribute); centralRepoCommentDialog.display(); if (centralRepoCommentDialog.isCommentUpdated()) { @@ -106,43 +99,11 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { } } catch (EamDbException ex) { logger.log(Level.SEVERE, "Error adding comment", ex); + //DLG: Create error popup dialog here. + return centralRepoCommentDialog.getOriginalComment(); } } - return centralRepoCommentDialog.getComment(); - } - - /** - * Create an instance labeled "Add/Edit Central Repository Comment" given an - * AbstractFile. This is intended for the result view. - * - * @param file The file from which a correlation attribute to modify is - * derived. - * - * @return The instance. - * - * @throws EamDbException - * @throws NoCurrentCaseException - * @throws TskCoreException - */ - @Messages({"AddEditCentralRepoCommentAction.menuItemText.addEditCentralRepoComment=Add/Edit Central Repository Comment"}) - public static AddEditCentralRepoCommentAction createAddEditCentralRepoCommentAction(AbstractFile file) { - - return new AddEditCentralRepoCommentAction(file, - Bundle.AddEditCentralRepoCommentAction_menuItemText_addEditCentralRepoComment()); - } - - /** - * Create an instance labeled "Add/Edit Comment" given a - * CorrelationAttribute. This is intended for the content view. - * - * @param correlationAttribute The correlation attribute to modify. - * - * @return The instance. - */ - @Messages({"AddEditCentralRepoCommentAction.menuItemText.addEditComment=Add/Edit Comment"}) - public static AddEditCentralRepoCommentAction createAddEditCommentAction(CorrelationAttribute correlationAttribute) { - - return new AddEditCentralRepoCommentAction(correlationAttribute, - Bundle.AddEditCentralRepoCommentAction_menuItemText_addEditComment()); + + return centralRepoCommentDialog.getNewComment(); } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java index c95b0bfde7..88708be173 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.centralrepository; +import org.openide.util.NbBundle.Messages; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; @@ -26,21 +27,22 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns * Dialog to allow Central Repository file instance comments to be added and * modified. */ +@Messages({"CentralRepoCommentDialog.title.addEditCentralRepoComment=Add/Edit Central Repository Comment"}) @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives final class CentralRepoCommentDialog extends javax.swing.JDialog { private final CorrelationAttribute correlationAttribute; private boolean commentUpdated = false; private String currentComment = ""; + private String originalComment = ""; /** * Create an instance. * * @param correlationAttribute The correlation attribute to be modified. - * @param title The title to assign the dialog. */ - CentralRepoCommentDialog(CorrelationAttribute correlationAttribute, String title) { - super(WindowManager.getDefault().getMainWindow(), title); + CentralRepoCommentDialog(CorrelationAttribute correlationAttribute) { + super(WindowManager.getDefault().getMainWindow(), Bundle.CentralRepoCommentDialog_title_addEditCentralRepoComment()); initComponents(); @@ -49,6 +51,7 @@ final class CentralRepoCommentDialog extends javax.swing.JDialog { // Store the original comment if (instance.getComment() != null) { currentComment = instance.getComment(); + originalComment = currentComment; } pathLabel.setText(instance.getFilePath()); @@ -82,11 +85,21 @@ final class CentralRepoCommentDialog extends javax.swing.JDialog { * Get the current comment. * If the user hit OK, this will be the new comment. * If the user canceled, this will be the original comment. + * * @return the comment */ - String getComment() { + String getNewComment() { return currentComment; } + + /** + * Get the original comment. + * + * @return the comment + */ + String getOriginalComment() { + return originalComment; + } /** * This method is called from within the constructor to initialize the form. diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java index 5a6e8fa652..dff9be8f91 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java @@ -47,7 +47,7 @@ public class CentralRepoContextMenuActionsProvider implements ContextMenuActions for (AbstractFile file : selectedFiles) { if (EamDbUtil.useCentralRepo() && EamArtifactUtil.isSupportedAbstractFileType(file) && file.isFile()) { - actions.add(AddEditCentralRepoCommentAction.createAddEditCentralRepoCommentAction(file)); + actions.add(new AddEditCentralRepoCommentAction(file)); } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties index b3375cda0d..acb3fa42d1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties @@ -3,7 +3,7 @@ DataContentViewerOtherCases.showCaseDetailsMenuItem.text=Show Case Details DataContentViewerOtherCases.table.toolTip.text=Click column name to sort. Right-click on the table for more options. DataContentViewerOtherCases.exportToCSVMenuItem.text=Export Selected Rows to CSV DataContentViewerOtherCases.showCommonalityMenuItem.text=Show Frequency -DataContentViewerOtherCases.addCommentMenuItem.text=Add/Edit Comment +DataContentViewerOtherCases.addCommentMenuItem.text=Add/Edit Central Repository Comment DataContentViewerOtherCases.earliestCaseDate.text=Earliest Case Date DataContentViewerOtherCases.earliestCaseLabel.toolTipText= DataContentViewerOtherCases.earliestCaseLabel.text=Central Repository Starting Date: diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 8ead475d95..2a5a98cd7e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -38,7 +38,6 @@ import java.util.Map; import java.util.Objects; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import java.util.stream.Collectors; import javax.swing.JFileChooser; import javax.swing.JMenuItem; import javax.swing.JOptionPane; @@ -127,12 +126,12 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi } else if (jmi.equals(addCommentMenuItem)) { try { OtherOccurrenceNodeData selectedNode = (OtherOccurrenceNodeData) tableModel.getRow(otherCasesTable.getSelectedRow()); - AddEditCentralRepoCommentAction action = AddEditCentralRepoCommentAction.createAddEditCommentAction(selectedNode.createCorrelationAttribute()); + AddEditCentralRepoCommentAction action = new AddEditCentralRepoCommentAction(selectedNode.createCorrelationAttribute()); String currentComment = action.addEditCentralRepoComment(); selectedNode.updateComment(currentComment); otherCasesTable.repaint(); } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error performing Add/Edit Comment action", ex); + logger.log(Level.SEVERE, "Error performing Add/Edit Central Repository Comment action", ex); } } } From 23bc1d632ac8d5952c29524a48940029f76df865 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Jul 2018 16:13:57 -0400 Subject: [PATCH 44/96] Improvement to case log when handling archive data sources --- .../autopsy/experimental/autoingest/AddArchiveTask.java | 3 +-- .../autopsy/experimental/autoingest/AutoIngestManager.java | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 9830cecbe9..ba554e2309 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -33,7 +33,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.LocalDiskDSProcessor; import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; @@ -180,7 +179,7 @@ class AddArchiveTask implements Runnable { synchronized (archiveDspLock) { UUID taskId = UUID.randomUUID(); currentCase.notifyAddingDataSource(taskId); - AutoIngestDataSource internalDataSource = new AutoIngestDataSource(deviceId, newFilePath); + AutoIngestDataSource internalDataSource = new AutoIngestDataSource(newFilePath.getFileName().toString(), newFilePath); DataSourceProcessorCallback internalArchiveDspCallBack = new AddDataSourceCallback(currentCase, internalDataSource, taskId, archiveDspLock); selectedProcessor.process(deviceId, newFilePath, progressMonitor, internalArchiveDspCallBack); archiveDspLock.wait(); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 77ddcf8d63..65b4b5a5ce 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2790,6 +2790,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { + AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); if (!snapshot.isCancelled()) { List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); if (!cancelledModules.isEmpty()) { @@ -2798,15 +2799,15 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen setCaseNodeDataErrorsOccurred(caseDirectoryPath); for (String module : snapshot.getCancelledDataSourceIngestModules()) { sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); - jobLogger.logIngestModuleCancelled(module); + nestedJobLogger.logIngestModuleCancelled(module); } } - jobLogger.logAnalysisCompleted(); + nestedJobLogger.logAnalysisCompleted(); } else { currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); currentJob.setErrorsOccurred(true); setCaseNodeDataErrorsOccurred(caseDirectoryPath); - jobLogger.logAnalysisCancelled(); + nestedJobLogger.logAnalysisCancelled(); CancellationReason cancellationReason = snapshot.getCancellationReason(); if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); From b339a09a2628db52f47e08d8cf7f8156d5c09cde Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Jul 2018 16:19:40 -0400 Subject: [PATCH 45/96] Minor --- .../autopsy/experimental/autoingest/AddArchiveTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index ba554e2309..e62c58343c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -179,7 +179,7 @@ class AddArchiveTask implements Runnable { synchronized (archiveDspLock) { UUID taskId = UUID.randomUUID(); currentCase.notifyAddingDataSource(taskId); - AutoIngestDataSource internalDataSource = new AutoIngestDataSource(newFilePath.getFileName().toString(), newFilePath); + AutoIngestDataSource internalDataSource = new AutoIngestDataSource(deviceId, newFilePath); DataSourceProcessorCallback internalArchiveDspCallBack = new AddDataSourceCallback(currentCase, internalDataSource, taskId, archiveDspLock); selectedProcessor.process(deviceId, newFilePath, progressMonitor, internalArchiveDspCallBack); archiveDspLock.wait(); From fe32cb5500d1d5bc9f9fe20741df3b711acd9d44 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 6 Jul 2018 17:24:45 -0400 Subject: [PATCH 46/96] Update SearchEngineURLQueryAnalyzer.java --- .../autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index 7bde8113a8..4c8999ec47 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -245,7 +245,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS return x; } catch (IllegalArgumentException exception) { //if it fails, return the encoded string - logger.log(Level.SEVERE, "Illegal arguement Exception during URL decoding, returning undecoded value:" + logger.log(Level.SEVERE, "Illegal argument passed to URL decoding, returning undecoded value:" + "\n\tURL: " + url + "\n\tUndecoded value: " + x + "\n\tEngine name: " + eng.getEngineName() From 1912be53e56988ecd4502e98d956b8abce33bd7f Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Sat, 7 Jul 2018 00:35:14 -0400 Subject: [PATCH 47/96] Improved error handling. --- .../AddEditCentralRepoCommentAction.java | 47 ++++++++++++------- .../DataContentViewerOtherCases.java | 9 ++-- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index 90317ab747..8bca85ac70 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -21,6 +21,9 @@ package org.sleuthkit.autopsy.centralrepository; import java.awt.event.ActionEvent; import java.util.logging.Level; import javax.swing.AbstractAction; +import org.openide.DialogDisplayer; +import org.openide.NotifyDescriptor; +import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; @@ -40,6 +43,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { private boolean addToDatabase; private CorrelationAttribute correlationAttribute; + private String comment; /** * Private constructor to create an instance given a CorrelationAttribute. @@ -54,8 +58,8 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { /** * Private constructor to create an instance given an AbstractFile. * - * @param file The file from which a correlation attribute to modify is - * derived. + * @param file The file from which a correlation attribute to modify is + * derived. */ public AddEditCentralRepoCommentAction(AbstractFile file) { super(Bundle.AddEditCentralRepoCommentAction_menuItemText_addEditCentralRepoComment()); @@ -66,26 +70,24 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { } } - @Override - public void actionPerformed(ActionEvent event) { - addEditCentralRepoComment(); - } - /** * Create a Add/Edit dialog for the correlation attribute file instance * comment. The comment will be updated in the database if the file instance * exists there, or a new file instance will be added to the database with * the comment attached otherwise. - * - * The current comment for this instance is returned in case it is needed to - * update the display. * - * @return the current comment for this instance + * The current comment for this instance is is saved in case it is needed to + * update the display. If the comment was not changed either due to the + * action being canceled or the occurrence of an error, the comment will be + * null. */ - public String addEditCentralRepoComment() { + @Override + public void actionPerformed(ActionEvent event) { CentralRepoCommentDialog centralRepoCommentDialog = new CentralRepoCommentDialog(correlationAttribute); centralRepoCommentDialog.display(); + comment = null; + if (centralRepoCommentDialog.isCommentUpdated()) { EamDb dbManager; @@ -97,13 +99,26 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { } else { dbManager.updateAttributeInstanceComment(correlationAttribute); } + + comment = centralRepoCommentDialog.getNewComment(); } catch (EamDbException ex) { logger.log(Level.SEVERE, "Error adding comment", ex); - //DLG: Create error popup dialog here. - return centralRepoCommentDialog.getOriginalComment(); + NotifyDescriptor notifyDescriptor = new NotifyDescriptor.Message( + "An error occurred while trying to save the comment to the central repository.", + NotifyDescriptor.ERROR_MESSAGE); + DialogDisplayer.getDefault().notify(notifyDescriptor); } } - - return centralRepoCommentDialog.getNewComment(); + } + + /** + * Retrieve the comment that was last saved. If a comment update was + * canceled or an error occurred while attempting to save the comment, the + * comment will be null. + * + * @return The comment. + */ + public String getComment() { + return comment; } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 2a5a98cd7e..26ceed5500 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -127,9 +127,12 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi try { OtherOccurrenceNodeData selectedNode = (OtherOccurrenceNodeData) tableModel.getRow(otherCasesTable.getSelectedRow()); AddEditCentralRepoCommentAction action = new AddEditCentralRepoCommentAction(selectedNode.createCorrelationAttribute()); - String currentComment = action.addEditCentralRepoComment(); - selectedNode.updateComment(currentComment); - otherCasesTable.repaint(); + action.actionPerformed(null); + String currentComment = action.getComment(); + if (currentComment != null) { + selectedNode.updateComment(action.getComment()); + otherCasesTable.repaint(); + } } catch (EamDbException ex) { logger.log(Level.SEVERE, "Error performing Add/Edit Central Repository Comment action", ex); } From af56427c4421ea9ff8ddbf0cc6b7c1e72b4a914c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Sat, 7 Jul 2018 00:56:37 -0400 Subject: [PATCH 48/96] Fixed typos; minor code polish. --- .../AddEditCentralRepoCommentAction.java | 8 ++++---- .../centralrepository/CentralRepoCommentDialog.java | 13 +------------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index 8bca85ac70..af59d2754c 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -46,7 +46,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { private String comment; /** - * Private constructor to create an instance given a CorrelationAttribute. + * Constructor to create an instance given a CorrelationAttribute. * * @param correlationAttribute The correlation attribute to modify. */ @@ -56,7 +56,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { } /** - * Private constructor to create an instance given an AbstractFile. + * Constructor to create an instance given an AbstractFile. * * @param file The file from which a correlation attribute to modify is * derived. @@ -76,7 +76,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { * exists there, or a new file instance will be added to the database with * the comment attached otherwise. * - * The current comment for this instance is is saved in case it is needed to + * The current comment for this instance is saved in case it is needed to * update the display. If the comment was not changed either due to the * action being canceled or the occurrence of an error, the comment will be * null. @@ -100,7 +100,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { dbManager.updateAttributeInstanceComment(correlationAttribute); } - comment = centralRepoCommentDialog.getNewComment(); + comment = centralRepoCommentDialog.getComment(); } catch (EamDbException ex) { logger.log(Level.SEVERE, "Error adding comment", ex); NotifyDescriptor notifyDescriptor = new NotifyDescriptor.Message( diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java index 88708be173..529ffb8529 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoCommentDialog.java @@ -34,7 +34,6 @@ final class CentralRepoCommentDialog extends javax.swing.JDialog { private final CorrelationAttribute correlationAttribute; private boolean commentUpdated = false; private String currentComment = ""; - private String originalComment = ""; /** * Create an instance. @@ -51,7 +50,6 @@ final class CentralRepoCommentDialog extends javax.swing.JDialog { // Store the original comment if (instance.getComment() != null) { currentComment = instance.getComment(); - originalComment = currentComment; } pathLabel.setText(instance.getFilePath()); @@ -88,18 +86,9 @@ final class CentralRepoCommentDialog extends javax.swing.JDialog { * * @return the comment */ - String getNewComment() { + String getComment() { return currentComment; } - - /** - * Get the original comment. - * - * @return the comment - */ - String getOriginalComment() { - return originalComment; - } /** * This method is called from within the constructor to initialize the form. From 6b442c8f36126842b63f04f222bf43c55721f4bf Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 9 Jul 2018 13:44:14 -0400 Subject: [PATCH 49/96] Disable menu item when correlation attribute not available. --- .../AddEditCentralRepoCommentAction.java | 9 +++++++++ .../CentralRepoContextMenuActionsProvider.java | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index af59d2754c..df98197428 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -121,4 +121,13 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { public String getComment() { return comment; } + + /** + * Retrieve the associated correlation attribute. + * + * @return The correlation attribute. + */ + public CorrelationAttribute getCorrelationAttribute() { + return correlationAttribute; + } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java index dff9be8f91..65a980558c 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/CentralRepoContextMenuActionsProvider.java @@ -38,19 +38,23 @@ public class CentralRepoContextMenuActionsProvider implements ContextMenuActions @Override public List getActions() { - ArrayList actions = new ArrayList<>(); + ArrayList actionsList = new ArrayList<>(); Collection selectedFiles = Utilities.actionsGlobalContext().lookupAll(AbstractFile.class); if (selectedFiles.size() != 1) { - return actions; + return actionsList; } for (AbstractFile file : selectedFiles) { if (EamDbUtil.useCentralRepo() && EamArtifactUtil.isSupportedAbstractFileType(file) && file.isFile()) { - actions.add(new AddEditCentralRepoCommentAction(file)); + AddEditCentralRepoCommentAction action = new AddEditCentralRepoCommentAction(file); + if (action.getCorrelationAttribute() == null) { + action.setEnabled(false); + } + actionsList.add(action); } } - return actions; + return actionsList; } } From 08aa6ab8a73de925d65c2efa14d6b474c206bb2d Mon Sep 17 00:00:00 2001 From: esaunders Date: Mon, 9 Jul 2018 16:27:49 -0400 Subject: [PATCH 50/96] Publish a resumed message regardless of whether the JobProcessingTask is waiting for directory scan completion or not. --- .../experimental/autoingest/AutoIngestManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 65b4b5a5ce..565c97ca6d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -1840,13 +1840,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen */ setChanged(); notifyObservers(Event.RESUMED); - - /** - * Publish an event to let remote listeners know that the - * node has been resumed. - */ - eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); } + /** + * Publish an event to let remote listeners know that the node + * has been resumed. + */ + eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); + pauseLock.notifyAll(); } } From d7c93a4e8eede6b9150f3fd2252e5085836a6fa2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 9 Jul 2018 16:30:29 -0400 Subject: [PATCH 51/96] Removed stack trace for lack of MD5. --- .../datamodel/EamArtifactUtil.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 6aad75d38e..808b314af0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -235,21 +235,37 @@ public class EamArtifactUtil { return null; } - CorrelationAttribute correlationAttribute = null; - + CorrelationAttribute correlationAttribute; + CorrelationAttribute.Type type; + CorrelationCase correlationCase; + CorrelationDataSource correlationDataSource; + String value; + String filePath; + try { - CorrelationAttribute.Type type = EamDb.getInstance().getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); - CorrelationCase correlationCase = EamDb.getInstance().getCase(Case.getCurrentCaseThrows()); + type = EamDb.getInstance().getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); + correlationCase = EamDb.getInstance().getCase(Case.getCurrentCaseThrows()); if (null == correlationCase) { correlationCase = EamDb.getInstance().newCase(Case.getCurrentCaseThrows()); } - CorrelationDataSource correlationDataSource = CorrelationDataSource.fromTSKDataSource(correlationCase, file.getDataSource()); - String value = file.getMd5Hash(); - String filePath = (file.getParentPath() + file.getName()).toLowerCase(); - - correlationAttribute = EamDb.getInstance().getCorrelationAttribute(type, correlationCase, correlationDataSource, value, filePath); - } catch (TskCoreException | EamDbException | NoCurrentCaseException ex) { + correlationDataSource = CorrelationDataSource.fromTSKDataSource(correlationCase, file.getDataSource()); + value = file.getMd5Hash(); + filePath = (file.getParentPath() + file.getName()).toLowerCase(); + } catch (TskCoreException | EamDbException ex) { logger.log(Level.SEVERE, "Error retrieving correlation attribute.", ex); + return null; + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Case is closed.", ex); + return null; + } + + try { + correlationAttribute = EamDb.getInstance().getCorrelationAttribute(type, correlationCase, correlationDataSource, value, filePath); + } catch (EamDbException ex) { + logger.log(Level.WARNING, String.format( + "Correlation attribute could not be retrieved for '%s' (id=%d): %s", + content.getName(), content.getId(), ex.getMessage())); + return null; } return correlationAttribute; @@ -300,9 +316,12 @@ public class EamArtifactUtil { af.getParentPath() + af.getName()); eamArtifact.addInstance(cei); return eamArtifact; - } catch (TskCoreException | EamDbException | NoCurrentCaseException ex) { + } catch (TskCoreException | EamDbException ex) { logger.log(Level.SEVERE, "Error making correlation attribute.", ex); return null; + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Case is closed.", ex); + return null; } } From aabab0689877e90a4dd33ef6569813f135398459 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 10 Jul 2018 11:57:12 -0400 Subject: [PATCH 52/96] Set UI selections to null before closing top components. --- .../CoreComponentControl.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java index bf26dc8295..06ad27962a 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java @@ -88,7 +88,21 @@ final public class CoreComponentControl { TopComponent directoryTree = null; TopComponent favorites = null; final WindowManager windowManager = WindowManager.getDefault(); + + // Set the UI selections to null before closing the top components. + // Otherwise it may experience errors trying to load data for the closed case. for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { + if(tc instanceof DataContent) { + ((DataContent) tc).setNode(null); + } else if(tc instanceof DataResult) { + ((DataResult) tc).setNode(null); + } + } + } + + for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { String tcName = tc.getName(); @@ -105,7 +119,7 @@ final public class CoreComponentControl { } } } - + if (directoryTree != null) { directoryTree.close(); } From c3e663e66e4f0f027259c9e5bd097efb3c8a7e2a Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 10 Jul 2018 12:50:39 -0400 Subject: [PATCH 53/96] Fix split regex --- .../src/org/sleuthkit/autopsy/recentactivity/Util.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index a47d33f22a..e2ee2248d1 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -86,11 +86,15 @@ class Util { public static String getBaseDomain(String url) { String host = null; + + if(url.contains("file/Documents%20and%20Settings")) { + System.out.println("Long one: " + url); + } //strip protocol String cleanUrl = url.replaceFirst("/.*:\\/\\//", ""); //strip after slashes - String dirToks[] = cleanUrl.split("/\\//"); + String dirToks[] = cleanUrl.split("\\/"); if (dirToks.length > 0) { host = dirToks[0]; } else { @@ -139,9 +143,10 @@ class Util { //was not a valid URL, try a less picky method if (result == null || result.trim().isEmpty()) { + System.out.println("getBaseDomain result: " + getBaseDomain(value)); return getBaseDomain(value); } - + System.out.println("extractDomain result: " + result); return result; } From 72394ee2b59ca9ab0e798460f118b93e7f75f435 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 10 Jul 2018 15:24:59 -0400 Subject: [PATCH 54/96] Removed excessive logging --- .../objectdetection/ObjectDetectectionFileIngestModule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index 3e58aac8c1..d6bef92ab8 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -129,7 +129,6 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter classifiers.get(classifierKey).detectMultiScale(originalImage, detectionRectangles); } catch (CvException ignored) { //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable - logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName() + " with object id of " + file.getId())); //NON-NLS continue; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented From f6b6aec267dd00ce2f432562d3633c06dd2bad44 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 11 Jul 2018 11:22:33 -0400 Subject: [PATCH 55/96] Fix protocol regex and increase max value length. --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 2 +- .../src/org/sleuthkit/autopsy/recentactivity/Util.java | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index b405965b6b..f535d58ccd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -59,7 +59,7 @@ abstract class AbstractSqlEamDb implements EamDb { private final Map> bulkArtifacts; // Maximum length for the value column in the instance tables - static final int MAX_VALUE_LENGTH = 128; + static final int MAX_VALUE_LENGTH = 256; // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index e2ee2248d1..f7a79b991d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -87,11 +87,8 @@ class Util { public static String getBaseDomain(String url) { String host = null; - if(url.contains("file/Documents%20and%20Settings")) { - System.out.println("Long one: " + url); - } //strip protocol - String cleanUrl = url.replaceFirst("/.*:\\/\\//", ""); + String cleanUrl = url.replaceFirst(".*:\\/\\/", ""); //strip after slashes String dirToks[] = cleanUrl.split("\\/"); From a72527a4d512fd3c037ccdd8f66be38ef1f2e131 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 11 Jul 2018 11:23:46 -0400 Subject: [PATCH 56/96] Cleanup --- .../src/org/sleuthkit/autopsy/recentactivity/Util.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index f7a79b991d..8b246b05aa 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -140,10 +140,8 @@ class Util { //was not a valid URL, try a less picky method if (result == null || result.trim().isEmpty()) { - System.out.println("getBaseDomain result: " + getBaseDomain(value)); return getBaseDomain(value); } - System.out.println("extractDomain result: " + result); return result; } From 94bdd8684021acc0e8e0ebf52fa98634d77a3873 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 11 Jul 2018 11:35:26 -0400 Subject: [PATCH 57/96] Don't bail out of saving settings if there is no case open --- .../ImageGalleryOptionsPanel.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java index 2b3fc1692f..7fbcaac1f0 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java @@ -203,17 +203,20 @@ final class ImageGalleryOptionsPanel extends javax.swing.JPanel { } void store() { - Case openCase; - try { - openCase = Case.getCurrentCaseThrows(); - } catch (NoCurrentCaseException ex) { - Logger.getLogger(ImageGalleryOptionsPanel.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return; - } ImageGalleryPreferences.setEnabledByDefault(enabledByDefaultBox.isSelected()); ImageGalleryController.getDefault().setListeningEnabled(enabledForCaseBox.isSelected()); - new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); ImageGalleryPreferences.setGroupCategorizationWarningDisabled(groupCategorizationWarningBox.isSelected()); + + // If a case is open, save the per case setting + try { + Case openCase = Case.getCurrentCaseThrows(); + new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); + } catch (NoCurrentCaseException ex) { + // It's not an error if there's no case open + } + + + } /** From 953b3628e19a7cdd83c4b0bcc33ad9ee818d43c9 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Jul 2018 12:14:07 -0400 Subject: [PATCH 58/96] Kick off an immediate scan on resume. --- .../experimental/autoingest/AutoIngestManager.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 565c97ca6d..8dddd2a3e6 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -490,6 +490,14 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen break; case RESUME: resume(); + + /** + * Kick off an immediate scan so that the next pending job + * will be picked up sooner than having to wait for the + * InputDirScannerTask to run again. + */ + scanInputDirsNow(); + break; case SHUTDOWN: shutDown(); From 50e13eb31e997862fe01d188f987361e56766f60 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Jul 2018 14:16:16 -0400 Subject: [PATCH 59/96] Fix for threading issue related to the fact that returning copies of the pending, running and completed lists iterated over the collections without syncrhonization leading to the possibilty that the collections might be modified by another thread at the same time. --- .../autoingest/AutoIngestAdminActions.java | 2 +- .../autoingest/AutoIngestDashboard.java | 9 +- .../autoingest/AutoIngestJobsNode.java | 23 ++-- .../autoingest/AutoIngestJobsPanel.java | 2 +- .../autoingest/AutoIngestMonitor.java | 105 +++++++----------- .../AutoIngestNodeRefreshEvents.java | 31 +++--- .../autoingest/PrioritizationAction.java | 8 +- 7 files changed, 79 insertions(+), 101 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java index 87c805fa12..1e1b82d8e7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java @@ -338,7 +338,7 @@ final class AutoIngestAdminActions { dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); AutoIngestManager.CaseDeletionResult result = dashboard.getMonitor().deleteCase(job); - dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor().getJobsSnapshot())); + dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor())); dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); if (AutoIngestManager.CaseDeletionResult.FAILED == result) { JOptionPane.showMessageDialog(dashboard, diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index 49e9cdbd17..031aa64daf 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -38,7 +38,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.core.ServicesMonitor; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeRefreshEvents.RefreshChildrenEvent; /** @@ -257,7 +256,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { @Override public void update(Observable observable, Object arg) { - if (arg instanceof JobsSnapshot) { + if (arg == null ) { EventQueue.invokeLater(() -> { refreshTables(); }); @@ -271,9 +270,9 @@ final class AutoIngestDashboard extends JPanel implements Observer { * @param nodeStateSnapshot The jobs snapshot. */ void refreshTables() { - pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); + pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); } /** diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java index a7b2ad6116..552e44834c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java @@ -33,7 +33,6 @@ import org.openide.nodes.Node; import org.openide.nodes.Sheet; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.datamodel.NodeProperty; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; import org.sleuthkit.autopsy.guiutils.DurationCellRenderer; import org.sleuthkit.autopsy.guiutils.StatusIconCellRenderer; @@ -61,13 +60,13 @@ final class AutoIngestJobsNode extends AbstractNode { /** * Construct a new AutoIngestJobsNode. * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which will be used to send and receive * refresh events */ - AutoIngestJobsNode(JobsSnapshot jobsSnapshot, AutoIngestJobStatus status, EventBus eventBus) { - super(Children.create(new AutoIngestNodeChildren(jobsSnapshot, status, eventBus), false)); + AutoIngestJobsNode(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + super(Children.create(new AutoIngestNodeChildren(monitor, status, eventBus), false)); refreshChildrenEventBus = eventBus; } @@ -84,7 +83,7 @@ final class AutoIngestJobsNode extends AbstractNode { static final class AutoIngestNodeChildren extends ChildFactory { private final AutoIngestJobStatus autoIngestJobStatus; - private JobsSnapshot jobsSnapshot; + private AutoIngestMonitor monitor; private final RefreshChildrenSubscriber refreshChildrenSubscriber = new RefreshChildrenSubscriber(); private final EventBus refreshEventBus; @@ -92,13 +91,13 @@ final class AutoIngestJobsNode extends AbstractNode { * Create children nodes for the AutoIngestJobsNode which will each * represent a single AutoIngestJob * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which the class registers to for * refresh events */ - AutoIngestNodeChildren(JobsSnapshot snapshot, AutoIngestJobStatus status, EventBus eventBus) { - jobsSnapshot = snapshot; + AutoIngestNodeChildren(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + this.monitor = monitor; autoIngestJobStatus = status; refreshEventBus = eventBus; refreshChildrenSubscriber.register(refreshEventBus); @@ -109,14 +108,14 @@ final class AutoIngestJobsNode extends AbstractNode { List jobs; switch (autoIngestJobStatus) { case PENDING_JOB: - jobs = jobsSnapshot.getPendingJobs(); + jobs = monitor.getPendingJobs(); Collections.sort(jobs); break; case RUNNING_JOB: - jobs = jobsSnapshot.getRunningJobs(); + jobs = monitor.getRunningJobs(); break; case COMPLETED_JOB: - jobs = jobsSnapshot.getCompletedJobs(); + jobs = monitor.getCompletedJobs(); break; default: jobs = new ArrayList<>(); @@ -167,7 +166,7 @@ final class AutoIngestJobsNode extends AbstractNode { //Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus //RefreshChildrenEvents can change which children are present however //RefreshJobEvents and RefreshCaseEvents can still change the order we want to display them in - jobsSnapshot = refreshEvent.getJobsSnapshot(); + monitor = refreshEvent.getMonitor(); refresh(true); } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java index a3cdc1c51d..fb6f79ab0c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java @@ -171,7 +171,7 @@ final class AutoIngestJobsPanel extends javax.swing.JPanel implements ExplorerMa ((AutoIngestJobsNode) explorerManager.getRootContext()).refresh(refreshEvent); } else { //Make a new AutoIngestJobsNode with it's own EventBus and set it as the root context - explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getJobsSnapshot(), status, new EventBus("AutoIngestJobsNodeEventBus"))); + explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getMonitor(), status, new EventBus("AutoIngestJobsNodeEventBus"))); } outline.setRowSelectionAllowed(true); outline.setFocusable(true); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index d9fb71f56c..1b1b1bd613 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -172,7 +172,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(event.getJob()); jobsSnapshot.addOrReplaceRunningJob(event.getJob()); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -190,7 +190,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(job); // Update the state of the existing job in the running jobs table - for (AutoIngestJob runningJob : jobsSnapshot.getRunningJobs()) { + for (AutoIngestJob runningJob : getRunningJobs()) { if (runningJob.equals(job)) { runningJob.setIngestJobsSnapshot(job.getIngestJobSnapshots()); runningJob.setIngestThreadSnapshot(job.getIngestThreadActivitySnapshots()); @@ -200,7 +200,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } } setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -216,7 +216,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removeRunningJob(job); jobsSnapshot.addOrReplaceCompletedJob(job); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -259,15 +259,35 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * Gets the auto ingest monitor's current snapshot of the pending jobs - * queue, running jobs list, and completed jobs list for an auto ingest - * cluster. + * Gets the snapshot of the pending jobs queue for an auto ingest cluster. * - * @return The snapshot. + * @return The pending jobs queue. */ - JobsSnapshot getJobsSnapshot() { + List getPendingJobs() { synchronized (jobsLock) { - return jobsSnapshot; + return new ArrayList<>(jobsSnapshot.pendingJobs); + } + } + + /** + * Gets the snapshot of the running jobs list for an auto ingest cluster. + * + * @return The running jobs list. + */ + List getRunningJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.runningJobs); + } + } + + /** + * Gets the snapshot of the completed jobs list for an auto ingest cluster. + * + * @return The completed jobs list. + */ + List getCompletedJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.completedJobs); } } @@ -287,10 +307,9 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * * @return The refreshed snapshot. */ - JobsSnapshot refreshJobsSnapshot() { + void refreshJobsSnapshot() { synchronized (jobsLock) { jobsSnapshot = queryCoordinationService(); - return jobsSnapshot; } } @@ -358,13 +377,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeCase(final String caseName) throws AutoIngestMonitorException { + void deprioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToDeprioritize = new ArrayList<>(); synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getManifest().getCaseName().equals(caseName)) { jobsToDeprioritize.add(pendingJob); } @@ -395,7 +413,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_DEPRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -407,13 +424,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeCase(final String caseName) throws AutoIngestMonitorException { + void prioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToPrioritize = new ArrayList<>(); int highestPriority = 0; synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -448,7 +464,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_PRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -460,15 +475,14 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { AutoIngestJob jobToDeprioritize = null; /* * Make sure the job is still in the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.equals(job)) { jobToDeprioritize = job; break; @@ -506,7 +520,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen }).start(); } - return jobsSnapshot; } } @@ -518,9 +531,8 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { int highestPriority = 0; AutoIngestJob jobToPrioritize = null; @@ -528,7 +540,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * Get the highest known priority and make sure the job is still in * the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -569,7 +581,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen }).start(); } - return jobsSnapshot; } } @@ -591,7 +602,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ void reprocessJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { - if (!jobsSnapshot.getCompletedJobs().contains(job)) { + if (!getCompletedJobs().contains(job)) { return; } @@ -660,7 +671,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen // Update the state of completed jobs associated with this case to indicate // that the case has been deleted - for (AutoIngestJob completedJob : jobsSnapshot.getCompletedJobs()) { + for (AutoIngestJob completedJob : getCompletedJobs()) { if (caseName.equals(completedJob.getManifest().getCaseName())) { try { completedJob.setProcessingStatus(DELETED); @@ -741,7 +752,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen synchronized (jobsLock) { jobsSnapshot = queryCoordinationService(); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } } @@ -752,42 +763,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * A snapshot of the pending jobs queue, running jobs list and completed * jobs list for an auto ingest cluster. */ - static final class JobsSnapshot { + private static final class JobsSnapshot { private final Set pendingJobs = new HashSet<>(); private final Set runningJobs = new HashSet<>(); private final Set completedJobs = new HashSet<>(); - /** - * Gets the snapshot of the pending jobs queue for an auto ingest - * cluster. - * - * @return The pending jobs queue. - */ - List getPendingJobs() { - return new ArrayList<>(this.pendingJobs); - } - - /** - * Gets the snapshot of the running jobs list for an auto ingest - * cluster. - * - * @return The running jobs list. - */ - List getRunningJobs() { - return new ArrayList<>(this.runningJobs); - } - - /** - * Gets the snapshot of the completed jobs list for an auto ingest - * cluster. - * - * @return The completed jobs list. - */ - List getCompletedJobs() { - return new ArrayList<>(this.completedJobs); - } - /** * Adds an auto job to the snapshot of the pending jobs queue for an * auto ingest cluster. If an equivalent job already exists, it is diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java index 2c554bf00f..338bce31c7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.experimental.autoingest; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; - /** * Class which contains events to identify what should be refreshed in the * AutoIngestJobsNode @@ -31,19 +29,20 @@ class AutoIngestNodeRefreshEvents { */ static class AutoIngestRefreshEvent { - private final JobsSnapshot jobsSnapshot; + private final AutoIngestMonitor monitor; - AutoIngestRefreshEvent(JobsSnapshot jobs) { - this.jobsSnapshot = jobs; + AutoIngestRefreshEvent(AutoIngestMonitor monitor) { + this.monitor = monitor; } /** - * Get the state of the jobs lists when the event was fired. + * Get the monitor which will provide access to the state of + * the jobs. * * @return */ - JobsSnapshot getJobsSnapshot() { - return this.jobsSnapshot; + AutoIngestMonitor getMonitor() { + return this.monitor; } } @@ -56,8 +55,8 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshChildrenEvent. */ - RefreshChildrenEvent(JobsSnapshot jobs) { - super(jobs); + RefreshChildrenEvent(AutoIngestMonitor monitor) { + super(monitor); } } @@ -72,11 +71,11 @@ class AutoIngestNodeRefreshEvents { /** * Contructs a RefreshCaseEvent * - * @param jobs The current state of the jobs lists. + * @param monitor The monitor that will provide access to the current state of the jobs lists. * @param name The name of the case whose nodes should be refreshed. */ - RefreshCaseEvent(JobsSnapshot jobs, String name) { - super(jobs); + RefreshCaseEvent(AutoIngestMonitor monitor, String name) { + super(monitor); caseName = name; } @@ -103,11 +102,11 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshJobEvent. * - * @param jobs The curent state of the jobs lists. + * @param monitor The monitor which will provide access to the current state of the jobs lists. * @param job The job which should be refreshed. */ - RefreshJobEvent(JobsSnapshot jobs, AutoIngestJob job) { - super(jobs); + RefreshJobEvent(AutoIngestMonitor monitor, AutoIngestJob job) { + super(monitor); autoIngestJob = job; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java index 3d57057bff..006dc3e580 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java @@ -145,7 +145,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -184,7 +184,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -225,7 +225,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } @@ -266,7 +266,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } } From 3f89863a29d1b44e6273dcfbc4e5a454f62ff559 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 12 Jul 2018 00:40:07 -0400 Subject: [PATCH 60/96] Fix string format of error message --- InternalPythonModules/android/browserlocation.py | 2 +- InternalPythonModules/android/cachelocation.py | 2 +- InternalPythonModules/android/calllog.py | 2 +- InternalPythonModules/android/contact.py | 2 +- InternalPythonModules/android/googlemaplocation.py | 2 +- InternalPythonModules/android/tangomessage.py | 2 +- InternalPythonModules/android/textmessage.py | 2 +- InternalPythonModules/android/wwfmessage.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/InternalPythonModules/android/browserlocation.py b/InternalPythonModules/android/browserlocation.py index 6db230c2e6..a0d73fdd28 100644 --- a/InternalPythonModules/android/browserlocation.py +++ b/InternalPythonModules/android/browserlocation.py @@ -105,7 +105,7 @@ class BrowserLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactTypeName(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactTypeName()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/cachelocation.py b/InternalPythonModules/android/cachelocation.py index 019c03c654..53e60b51ee 100644 --- a/InternalPythonModules/android/cachelocation.py +++ b/InternalPythonModules/android/cachelocation.py @@ -138,7 +138,7 @@ class CacheLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/calllog.py b/InternalPythonModules/android/calllog.py index 79d0e82479..b20c0d0512 100644 --- a/InternalPythonModules/android/calllog.py +++ b/InternalPythonModules/android/calllog.py @@ -157,7 +157,7 @@ class CallLogAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index call log artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/contact.py b/InternalPythonModules/android/contact.py index 1d556607e0..22a1f797ec 100644 --- a/InternalPythonModules/android/contact.py +++ b/InternalPythonModules/android/contact.py @@ -164,7 +164,7 @@ class ContactAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index contact artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/googlemaplocation.py b/InternalPythonModules/android/googlemaplocation.py index 444c0477ee..712aacc2be 100644 --- a/InternalPythonModules/android/googlemaplocation.py +++ b/InternalPythonModules/android/googlemaplocation.py @@ -115,7 +115,7 @@ class GoogleMapLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS route artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/tangomessage.py b/InternalPythonModules/android/tangomessage.py index bd65738a6e..514bd48ecb 100644 --- a/InternalPythonModules/android/tangomessage.py +++ b/InternalPythonModules/android/tangomessage.py @@ -116,7 +116,7 @@ class TangoMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index Tango message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/textmessage.py b/InternalPythonModules/android/textmessage.py index a6b36ad549..5854977e8b 100644 --- a/InternalPythonModules/android/textmessage.py +++ b/InternalPythonModules/android/textmessage.py @@ -130,7 +130,7 @@ class TextMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index text message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/wwfmessage.py b/InternalPythonModules/android/wwfmessage.py index 446ab78b67..6b41ab2677 100644 --- a/InternalPythonModules/android/wwfmessage.py +++ b/InternalPythonModules/android/wwfmessage.py @@ -125,7 +125,7 @@ class WWFMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index WWF message artifact for keyword search.", artifact.getDisplayName()) From 677da001960a0f7638cfa6ab58268f2fc6e05a13 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 12 Jul 2018 13:20:20 -0400 Subject: [PATCH 61/96] Removed message. --- .../org/sleuthkit/autopsy/casemodule/services/Bundle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties index e9c1b5307c..266db3a64c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties @@ -1,6 +1,5 @@ OptionsCategory_Name_TagNamesOptions=Tags OptionsCategory_TagNames=TagNames -Blackboard.unableToIndexArtifact.error.msg=Unable to index blackboard artifact {0} TagNameDialog.title.text=New Tag TagNameDialog.JOptionPane.tagNameIllegalCharacters.message=Tag name may not contain any of the following symbols\: \\ \: * ? " < > | , ; TagNameDialog.JOptionPane.tagNameIllegalCharacters.title=Invalid character in tag name From 4b9159b22ebccb2f1cf5bb53e31c236bf0659dee Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 12 Jul 2018 15:01:17 -0400 Subject: [PATCH 62/96] Fix jitter in file types tree caused by missing space --- Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index 6a304d7b86..aa1d0a04ec 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -249,7 +249,7 @@ public final class FileTypes implements AutopsyVisitableItem { if (typesRoot.showCounts) { //only show "(counting...)" the first time, otherwise it is distracting. setDisplayName(getDisplayNameBase() + ((childCount < 0) ? Bundle.FileTypes_bgCounting_placeholder() - : ("(" + childCount + ")"))); //NON-NLS + : (" (" + childCount + ")"))); //NON-NLS new SwingWorker() { @Override protected Long doInBackground() throws Exception { From a50cf3da86b93788e884c10aa6feaa25f6268bf7 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 11:36:04 -0400 Subject: [PATCH 63/96] User searches appear in report (no heading yet) --- .../autopsy/report/TableReportGenerator.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 9d5d41c836..8c80955b23 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -555,6 +555,7 @@ class TableReportGenerator { "AND att.artifact_id = art.artifact_id " + //NON-NLS "GROUP BY list " + orderByClause; //NON-NLS + System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); @@ -589,8 +590,9 @@ class TableReportGenerator { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } // Query for keywords, grouped by list - String keywordsQuery - = "SELECT art.artifact_id, art.obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + // Query for keywords that are part of a list + String keywordListsQuery + = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + //NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, blackboard_attributes AS att3, tsk_files AS f " + //NON-NLS @@ -608,9 +610,26 @@ class TableReportGenerator { + //NON-NLS "AND (att3.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ") " + //NON-NLS - "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " - + //NON-NLS - orderByClause; //NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; + + // Query for keywords that are not part of a list + String keywordAdHocQuery = + "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS + "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS + "WHERE " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) " + // NON-NLS + "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) AND (att1.attribute_type_id = 37))) " + //NON-NLS + "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS + "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS + "AND (f.obj_id = art.obj_id) " + //NON-NLS + "AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID() + ") " + // NON-NLS + "AND (att2.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID() + ") " + // NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + + String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; + + System.out.println("\n\n### kw hits query:" + keywordsQuery); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); From 498c19c9dc6e4d2a9503d1ceaf99b083f137c133 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 13:20:45 -0400 Subject: [PATCH 64/96] Add fix for notable tags. --- .../autopsy/report/TableReportGenerator.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 8c80955b23..e8d8b067b8 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import static org.sleuthkit.autopsy.casemodule.services.TagsManager.getNotableTagLabel; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.Type; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -538,6 +540,32 @@ class TableReportGenerator { logger.log(Level.SEVERE, "Exception while getting open case: ", ex); //NON-NLS return; } + + // First check for ad-hoc keywords + String tagIDList = ""; + if(tagNamesFilter.isEmpty()) { + System.out.println("\n## no tags selected"); + } else { + System.out.println("\n## tags: " + makeCommaSeparatedList(tagNamesFilter)); + try { + Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); + for(String tagDisplayName : tagNamesFilter) { + if(tagNamesMap.containsKey(tagDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(tagDisplayName).getId(); + } else { + System.out.println("### No key for " + tagDisplayName); + } + } + } catch (NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, "Exception while getting tag info - proceeding without tag filter: ", ex); //NON-NLS + tagIDList = ""; + } + } + System.out.println("### tag ID list: " + tagIDList); + if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -546,15 +574,18 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art, blackboard_artifact_tags as tag " + //NON-NLS "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS - "AND att.artifact_id = art.artifact_id " - + //NON-NLS - "GROUP BY list " + orderByClause; //NON-NLS + "AND att.artifact_id = art.artifact_id "; + if (! tagIDList.isEmpty()) { + keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS + "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { @@ -1642,14 +1673,15 @@ class TableReportGenerator { private HashSet getUniqueTagNames(long artifactId) throws TskCoreException { HashSet uniqueTagNames = new HashSet<>(); - String query = "SELECT display_name, artifact_id FROM tag_names AS tn, blackboard_artifact_tags AS bat " + String query = "SELECT display_name, artifact_id, knownStatus FROM tag_names AS tn, blackboard_artifact_tags AS bat " + //NON-NLS "WHERE tn.tag_name_id = bat.tag_name_id AND bat.artifact_id = " + artifactId; //NON-NLS try (SleuthkitCase.CaseDbQuery dbQuery = Case.getCurrentCaseThrows().getSleuthkitCase().executeQuery(query)) { ResultSet tagNameRows = dbQuery.getResultSet(); while (tagNameRows.next()) { - uniqueTagNames.add(tagNameRows.getString("display_name")); //NON-NLS + String notableString = tagNameRows.getInt("knownStatus") == TskData.FileKnown.BAD.ordinal() ? getNotableTagLabel() : ""; + uniqueTagNames.add(tagNameRows.getString("display_name") + notableString); //NON-NLS } } catch (TskCoreException | SQLException | NoCurrentCaseException ex) { throw new TskCoreException("Error getting tag names for artifact: ", ex); From 478cf84e5d11d4ba74b71c56c30d171f06214dfe Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 13 Jul 2018 14:02:25 -0400 Subject: [PATCH 65/96] Fixed core window opening condition. --- .../sleuthkit/autopsy/casemodule/Case.java | 15 ++-- .../DirectoryTreeTopComponent.java | 87 ++++++++++--------- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index a9bbb39907..9019ae162d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -292,9 +292,10 @@ public class Case { */ ADDING_DATA_SOURCE_FAILED, /** - * A new data source has been added to the current case. The old value - * of the PropertyChangeEvent is null, the new value is the newly-added - * data source (type: Content). Cast the PropertyChangeEvent to + * A new data source or series of data sources have been added to the + * current case. The old value of the PropertyChangeEvent is null, the + * new value is the newly-added data source (type: Content). Cast the + * PropertyChangeEvent to * org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent to * access additional event data. */ @@ -1099,6 +1100,10 @@ public class Case { /* * Open the top components (windows within the main application * window). + * + * Note: If the core windows are not opened here, they will be + * opened via the DirectoryTreeTopComponent 'propertyChange()' + * method on a DATA_SOURCE_ADDED event. */ if (newCurrentCase.hasData()) { CoreComponentControl.openCoreWindows(); @@ -1351,11 +1356,11 @@ public class Case { public String getReportDirectory() { return getOrCreateSubdirectory(REPORTS_FOLDER); } - + /** * Gets the full path to the config directory for this case, creating it if * it does not exist. - * + * * @return The config directory path. */ public String getConfigDirectory() { diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 97b99103bd..bcbb72acb5 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -135,7 +135,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat this.forwardList = new LinkedList<>(); backButton.setEnabled(false); forwardButton.setEnabled(false); - + groupByDatasourceCheckBox.setSelected(UserPreferences.groupItemsInTreeByDatasource()); } @@ -159,7 +159,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } }); - + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.DATA_SOURCE_ADDED), this); this.em.addPropertyChangeListener(this); IngestManager.getInstance().addIngestJobEventListener(this); @@ -379,10 +379,11 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat public int getPersistenceType() { return TopComponent.PERSISTENCE_NEVER; } - + /** - * Ask the user if they want to group by data source when opening a large case. - * + * Ask the user if they want to group by data source when opening a large + * case. + * * @param currentCase * @param dataSourceCount */ @@ -410,7 +411,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // Save the response Properties props = new Properties(); - if(dialog.groupByDataSourceSelected()) { + if (dialog.groupByDataSourceSelected()) { props.setProperty("groupByDataSource", "true"); } else { props.setProperty("groupByDataSource", "false"); @@ -423,7 +424,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } } - + /** * Called only when top component was closed on all workspaces before and * now is opened for the first time on some workspace. The intent is to @@ -432,8 +433,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * here. */ @NbBundle.Messages({"# {0} - dataSourceCount", - "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", - "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) + "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) @Override public void componentOpened() { // change the cursor to "waiting cursor" for this operation @@ -462,21 +463,21 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } else { ModuleSettings.setConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME, String.valueOf(threshold)); } - + try { int dataSourceCount = currentCase.getDataSources().size(); - if (! UserPreferences.groupItemsInTreeByDatasource() && - dataSourceCount > threshold) { + if (!UserPreferences.groupItemsInTreeByDatasource() + && dataSourceCount > threshold) { promptForDataSourceGrouping(currentCase, dataSourceCount); } } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error loading data sources", ex); } } - + // if there's at least one image, load the image and open the top componen autopsyTreeChildrenFactory = new AutopsyTreeChildrenFactory(); - autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); + autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); Node root = new AbstractNode(autopsyTreeChildren) { //JIRA-2807: What is the point of these overrides? /** @@ -535,9 +536,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat tree.collapseNode(views); } /* - * JIRA-2806: What is this supposed to do? Right now it selects - * the data sources node, but the comment seems to indicate - * it is supposed to select the first datasource. + * JIRA-2806: What is this supposed to do? Right now it + * selects the data sources node, but the comment seems to + * indicate it is supposed to select the first datasource. */ // select the first image node, if there is one // (this has to happen after dataResult is opened, because the event @@ -566,7 +567,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // dataResult active) try { Node[] selections = get(); - if (selections != null && selections.length > 0){ + if (selections != null && selections.length > 0) { em.setSelectedNodes(selections); } } catch (PropertyVetoException ex) { @@ -674,15 +675,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } /** - * The "listener" that listens to any changes made in the Case.java class. - * It will do something based on the changes in the Case.java class. + * The "listener" that monitors changes made in the Case class. This serves + * the purpose of keeping the UI in sync with the data as it changes. * - * @param evt the property change event + * @param event The property change event. */ @Override - public void propertyChange(PropertyChangeEvent evt) { + public void propertyChange(PropertyChangeEvent event) { if (RuntimeProperties.runningWithGUI()) { - String changed = evt.getPropertyName(); + String changed = event.getPropertyName(); if (changed.equals(Case.Events.CURRENT_CASE.toString())) { // changed current case // When a case is closed, the old value of this property is the // closed Case object and the new value is null. When a case is @@ -692,15 +693,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // opened events instead of property change events would be a better // solution. Either way, more probably needs to be done to clean up // data model objects when a case is closed. - if (evt.getOldValue() != null && evt.getNewValue() == null) { + if (event.getOldValue() != null && event.getNewValue() == null) { // The current case has been closed. Reset the ExplorerManager. SwingUtilities.invokeLater(() -> { Node emptyNode = new AbstractNode(Children.LEAF); em.setRootContext(emptyNode); }); - } else if (evt.getNewValue() != null) { + } else if (event.getNewValue() != null) { // A new case has been opened. Reset the ExplorerManager. - Case newCase = (Case) evt.getNewValue(); + Case newCase = (Case) event.getNewValue(); final String newCaseName = newCase.getName(); SwingUtilities.invokeLater(() -> { em.getRootContext().setName(newCaseName); @@ -724,20 +725,27 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * already closed. */ try { - Case currentCase = Case.getCurrentCaseThrows(); - // We only need to trigger openCoreWindows() when the - // first data source is added. - if (currentCase.getDataSources().size() == 1) { + Case.getCurrentCaseThrows(); + /* + * In case the Case 'updateGUIForCaseOpened()' method hasn't + * already done so, open the tree and all other core + * windows. + * + * TODO: (JIRA-4053) DirectoryTreeTopComponent should not be + * responsible for opening core windows. Consider moving + * this to the Case class. + */ + if (!this.isOpened()) { SwingUtilities.invokeLater(CoreComponentControl::openCoreWindows); } - } catch (NoCurrentCaseException | TskCoreException notUsed) { + } catch (NoCurrentCaseException notUsed) { /** * Case is closed, do nothing. */ } } // change in node selection else if (changed.equals(ExplorerManager.PROP_SELECTED_NODES)) { - respondSelection((Node[]) evt.getOldValue(), (Node[]) evt.getNewValue()); + respondSelection((Node[]) event.getOldValue(), (Node[]) event.getNewValue()); } else if (changed.equals(IngestManager.IngestModuleEvent.DATA_ADDED.toString())) { // nothing to do here. // all nodes should be listening for these events and update accordingly. @@ -882,7 +890,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat /** * Rebuilds the autopsy tree. - * + * * Does nothing if there is no open case. */ private void rebuildTree() { @@ -895,9 +903,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return; } if (null == currentCase || currentCase.hasData() == false) { - return; + return; } - + // refresh all children of the root. autopsyTreeChildrenFactory.refreshChildren(); @@ -924,14 +932,14 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } }.execute(); } - + /** * Selects the first node in the tree. - * + * */ - private void selectFirstChildNode () { + private void selectFirstChildNode() { Children rootChildren = em.getRootContext().getChildren(); - + if (rootChildren.getNodesCount() > 0) { Node firstNode = rootChildren.getNodeAt(0); if (firstNode != null) { @@ -940,6 +948,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } } + /** * Set the selected node using a path to a previously selected node. * From 9ef33ab771bf8343d062b934fc6ee6938bf67d21 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 14:16:35 -0400 Subject: [PATCH 66/96] Added user searches to table of contents when applicable Added fix for Notable tags Cleanup --- .../autopsy/report/TableReportGenerator.java | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index e8d8b067b8..59951df1f9 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -541,12 +541,9 @@ class TableReportGenerator { return; } - // First check for ad-hoc keywords + // Get a list of all selected tag IDs String tagIDList = ""; - if(tagNamesFilter.isEmpty()) { - System.out.println("\n## no tags selected"); - } else { - System.out.println("\n## tags: " + makeCommaSeparatedList(tagNamesFilter)); + if( ! tagNamesFilter.isEmpty()) { try { Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); for(String tagDisplayName : tagNamesFilter) { @@ -556,7 +553,16 @@ class TableReportGenerator { } tagIDList += tagNamesMap.get(tagDisplayName).getId(); } else { - System.out.println("### No key for " + tagDisplayName); + // If the tag name ends with "(Notable)", try stripping that off + if(tagDisplayName.endsWith(getNotableTagLabel())) { + String editedDisplayName = tagDisplayName.substring(0, tagDisplayName.length() - getNotableTagLabel().length()); + if(tagNamesMap.containsKey(editedDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(editedDisplayName).getId(); + } + } } } } catch (NoCurrentCaseException | TskCoreException ex) { @@ -564,8 +570,35 @@ class TableReportGenerator { tagIDList = ""; } } - System.out.println("### tag ID list: " + tagIDList); + // Check if there are any ad-hoc results + String adHocCountQuery = "SELECT COUNT(*) FROM " + //NON-NLS + "(SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 ";//NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + adHocCountQuery += "WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += " AND (art.artifact_id = tag.artifact_id) AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + adHocCountQuery += "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ")) "; //NON-NLS + + int adHocCount = 0; + try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(adHocCountQuery)) { + ResultSet adHocCountResultSet = dbQuery.getResultSet(); + if (adHocCountResultSet.next()) { + adHocCount = adHocCountResultSet.getInt(1); //NON-NLS + } else { + throw new TskCoreException("Error counting ad hoc keywords"); + } + } catch (TskCoreException | SQLException ex) { + errorList.add(NbBundle.getMessage(this.getClass(), "ReportGenerator.errList.failedQueryKWLists")); + logger.log(Level.SEVERE, "Failed to count ad hoc searches with query " + adHocCountQuery, ex); //NON-NLS + return; + } + + // Create the query to get the keyword list names if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -585,9 +618,12 @@ class TableReportGenerator { keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS } + if (adHocCount > 0) { + keywordListQuery += " UNION SELECT \"\" AS list "; + } keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS - System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); + // Make the table of contents links for each list type try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); List lists = new ArrayList<>(); @@ -611,6 +647,7 @@ class TableReportGenerator { return; } + // Query for keywords, grouped by list if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att3.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + "convert_to(att1.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS @@ -620,7 +657,7 @@ class TableReportGenerator { } else { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } - // Query for keywords, grouped by list + // Query for keywords that are part of a list String keywordListsQuery = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " @@ -648,9 +685,9 @@ class TableReportGenerator { "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS "WHERE " + // NON-NLS - " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " + // NON-NLS "EXCEPT " + // NON-NLS - "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) AND (att1.attribute_type_id = 37))) " + //NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + "))) " + //NON-NLS "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS "AND (f.obj_id = art.obj_id) " + //NON-NLS @@ -659,8 +696,6 @@ class TableReportGenerator { "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; - - System.out.println("\n\n### kw hits query:" + keywordsQuery); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); From 414a65402aed9dc6e269ef469ca68fa7a9f8106b Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 14:30:21 -0400 Subject: [PATCH 67/96] Cleanup --- .../sleuthkit/autopsy/report/TableReportGenerator.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 59951df1f9..f87851fe87 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -607,9 +607,11 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art, blackboard_artifact_tags as tag " - + //NON-NLS - "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art "; // NON-NLS + if(! tagIDList.isEmpty()) { + keywordListQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + keywordListQuery += "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS From 710f129d00f9e0b15ffdba7ec40d49391d319554 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 13 Jul 2018 17:00:41 -0400 Subject: [PATCH 68/96] Update DirectoryTreeTopComponent.java --- .../autopsy/directorytree/DirectoryTreeTopComponent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index bcbb72acb5..95ba2f110c 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -733,7 +733,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * * TODO: (JIRA-4053) DirectoryTreeTopComponent should not be * responsible for opening core windows. Consider moving - * this to the Case class. + * this elsewhere. */ if (!this.isOpened()) { SwingUtilities.invokeLater(CoreComponentControl::openCoreWindows); From e3eb0918fbae9b21e28c68fb3182a34efe991b34 Mon Sep 17 00:00:00 2001 From: esaunders Date: Mon, 16 Jul 2018 11:02:59 -0400 Subject: [PATCH 69/96] Query node status every 5 minutes and only display nodes that have been heard from in the last 15 minutes. --- .../autoingest/AutoIngestMonitor.java | 60 ++++++++++++------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index 1b1b1bd613..876a56c47e 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -116,11 +117,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } catch (AutopsyEventException ex) { throw new AutoIngestMonitorException("Failed to open auto ingest event channel", ex); //NON-NLS } - coordSvcQueryExecutor.scheduleWithFixedDelay(new CoordinationServiceQueryTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); + coordSvcQueryExecutor.scheduleWithFixedDelay(new StateRefreshTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); eventPublisher.addSubscriber(EVENT_LIST, this); - // Publish an event that asks running nodes to send their state. - eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + refreshNodeState(); } /** @@ -226,7 +226,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case prioritization event. */ private void handleCasePrioritizationEvent(AutoIngestCasePrioritizedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -235,7 +235,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case deletion event. */ private void handleCaseDeletedEvent(AutoIngestCaseDeletedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -297,7 +297,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @return */ List getNodeStates() { - return nodeStates.values().stream().collect(Collectors.toList()); + // We only report the state for nodes for which we have received + // a 'state' event in the last 15 minutes. + return nodeStates.values() + .stream() + .filter(s -> s.getLastSeenTime().isAfter(Instant.now().minus(Duration.ofMinutes(15)))) + .collect(Collectors.toList()); } /** @@ -313,6 +318,14 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } } + /** + * Ask running auto ingest nodes to report their state. + */ + private void refreshNodeState() { + // Publish an event that asks running nodes to send their state. + eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + } + /** * Gets a new snapshot of the pending jobs queue, running jobs list, and * completed jobs list for an auto ingest cluster. @@ -735,25 +748,26 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * A task that queries the coordination service for auto ingest manifest - * node data and converts it to auto ingest jobs for publication top its - * observers. + * A task that updates the state maintained by the monitor. + * At present this includes auto ingest job and auto ingest node data. + * The job data is refreshed by querying the coordination service for + * auto ingest manifest nodes. + * The auto ingest node data is refreshed by publishing a message asking + * all nodes to report their state. */ - private final class CoordinationServiceQueryTask implements Runnable { + private final class StateRefreshTask implements Runnable { - /** - * Queries the coordination service for auto ingest manifest node data - * and converts it to auto ingest jobs for publication top its - * observers. - */ @Override public void run() { if (!Thread.currentThread().isInterrupted()) { - synchronized (jobsLock) { - jobsSnapshot = queryCoordinationService(); - setChanged(); - notifyObservers(); - } + // Query coordination service for jobs data. + refreshJobsSnapshot(); + + // Ask running auto ingest nodes to report their status. + refreshNodeState(); + + setChanged(); + notifyObservers(); } } @@ -867,6 +881,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen private final String nodeName; private final State nodeState; + private final Instant lastSeenTime; AutoIngestNodeState(String name, Event event) { nodeName = name; @@ -896,6 +911,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen nodeState = State.UNKNOWN; break; } + lastSeenTime = Instant.now(); } String getName() { @@ -905,6 +921,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen State getState() { return nodeState; } + + Instant getLastSeenTime() { + return lastSeenTime; + } } /** From 718d28e8cc22b7405d022581b4965d2475586b6a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 16 Jul 2018 12:02:55 -0400 Subject: [PATCH 70/96] Changed finishing message. --- Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java index 71404c76e7..4200620749 100644 --- a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java +++ b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java @@ -134,7 +134,7 @@ class ImageWriter implements PropertyChangeListener{ @Messages({ "# {0} - data source name", - "ImageWriter.progressBar.message=Finishing acquisition of {0}" + "ImageWriter.progressBar.message=Finishing acquisition of {0} (unplug device to cancel)" }) private void startFinishImage(String dataSourceName){ From 263a63fb7e436abffe4ccb876a40846ea44aef4e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 17 Jul 2018 16:03:32 -0400 Subject: [PATCH 71/96] Added '(Volatility)' to the Memory DSP name. --- .../autopsy/experimental/volatilityDSP/MemoryDSProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java index dc05cb8376..5dbdca94ce 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java @@ -56,7 +56,7 @@ public class MemoryDSProcessor implements DataSourceProcessor { * * @return A data source type display string for this data source processor. */ - @Messages({"MemoryDSProcessor.dataSourceType=Memory Image File"}) + @Messages({"MemoryDSProcessor.dataSourceType=Memory Image File (Volatility)"}) public static String getType() { return Bundle.MemoryDSProcessor_dataSourceType(); } From fbc91771d2aa8fd8e95cf87fd3ce94a54c35b957 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 18 Jul 2018 12:14:57 -0400 Subject: [PATCH 72/96] Set checkbox by default. --- Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java index cb1ec5ddfd..c5c8fb32fb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java @@ -257,6 +257,7 @@ final class LocalDiskPanel extends JPanel { pathTextField.setEnabled(copyImageCheckbox.isSelected()); browseButton.setEnabled(copyImageCheckbox.isSelected()); changeDatabasePathCheckbox.setEnabled(copyImageCheckbox.isSelected()); + changeDatabasePathCheckbox.setSelected(copyImageCheckbox.isSelected()); fireUpdateEvent(); }//GEN-LAST:event_copyImageCheckboxActionPerformed From 60446f04066a0138d8f5f534e75256c19af33c13 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 18 Jul 2018 13:47:07 -0400 Subject: [PATCH 73/96] Added VHD to picker. --- .../org/sleuthkit/autopsy/casemodule/MissingImageDialog.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java b/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java index 5645e1d5d6..78cef08b25 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java @@ -46,11 +46,13 @@ class MissingImageDialog extends javax.swing.JDialog { SleuthkitCase db; static final GeneralFilter rawFilter = new GeneralFilter(GeneralFilter.RAW_IMAGE_EXTS, GeneralFilter.RAW_IMAGE_DESC); static final GeneralFilter encaseFilter = new GeneralFilter(GeneralFilter.ENCASE_IMAGE_EXTS, GeneralFilter.ENCASE_IMAGE_DESC); + static final GeneralFilter virtualMachineFilter = new GeneralFilter(GeneralFilter.VIRTUAL_MACHINE_EXTS, GeneralFilter.VIRTUAL_MACHINE_DESC); static final List allExt = new ArrayList(); static { allExt.addAll(GeneralFilter.RAW_IMAGE_EXTS); allExt.addAll(GeneralFilter.ENCASE_IMAGE_EXTS); + allExt.addAll(GeneralFilter.VIRTUAL_MACHINE_EXTS); } static final String allDesc = NbBundle.getMessage(MissingImageDialog.class, "MissingImageDialog.allDesc.text"); static final GeneralFilter allFilter = new GeneralFilter(allExt, allDesc); @@ -74,6 +76,7 @@ class MissingImageDialog extends javax.swing.JDialog { fc.addChoosableFileFilter(rawFilter); fc.addChoosableFileFilter(encaseFilter); + fc.addChoosableFileFilter(virtualMachineFilter); fc.setFileFilter(allFilter); selectButton.setEnabled(false); From dceb7e884eb47c2fa687404980f94997bef660b3 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 18 Jul 2018 16:01:57 -0400 Subject: [PATCH 74/96] The use of AutoIngestJob as a node key in the dashboard caused problems because (a) the underlying job objects could be changed in other threads and (b) the equals() implementation only considered the manifest file path.New AutoIngestWrapper class was introduced to be used as a key. --- .../autoingest/AinStatusPanel.java | 12 -- .../autoingest/AutoIngestJobsNode.java | 158 ++++++++++++--- .../autoingest/AutoIngestMonitor.java | 183 +++++++++--------- 3 files changed, 219 insertions(+), 134 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java index 1cc10fab76..d744abe8b4 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java @@ -125,18 +125,6 @@ final class AinStatusPanel extends javax.swing.JPanel implements ExplorerManager setLayout(new java.awt.BorderLayout()); }// //GEN-END:initComponents - /** - * Get the AutoIngestJob for the currently selected node of this panel. - * - * @return AutoIngestJob which is currently selected in this panel - */ - AutoIngestJob getSelectedAutoIngestJob() { - Node[] selectedRows = explorerManager.getSelectedNodes(); - if (selectedRows.length == 1) { - return ((JobNode) selectedRows[0]).getAutoIngestJob(); - } - return null; - } // Variables declaration - do not modify//GEN-BEGIN:variables // End of variables declaration//GEN-END:variables diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java index 552e44834c..d5bfbd13d8 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Objects; import org.openide.nodes.AbstractNode; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; @@ -33,8 +34,10 @@ import org.openide.nodes.Node; import org.openide.nodes.Sheet; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.datamodel.NodeProperty; +import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.Stage; import org.sleuthkit.autopsy.guiutils.DurationCellRenderer; import org.sleuthkit.autopsy.guiutils.StatusIconCellRenderer; +import org.sleuthkit.autopsy.ingest.DataSourceIngestJob; /** * A node which represents all AutoIngestJobs of a given AutoIngestJobStatus. @@ -77,10 +80,103 @@ final class AutoIngestJobsNode extends AbstractNode { refreshChildrenEventBus.post(refreshEvent); } + /** + * The AutoIngestJob class considers auto ingest jobs to be equal if they + * have the same manifest path. This is not sufficient for the purposes of + * determining when the state of a job has changed. This class is used to + * distinguish between different auto ingest jobs based on the manifest + * path, the processing stage, the job snapshot and priority. + */ + private static final class AutoIngestJobWrapper implements Comparable { + + private final AutoIngestJob autoIngestJob; + + /** + * We keep our own references to the following job attributes because + * they can be changed by events in other threads which + */ + private final Stage jobStage; + private final List jobSnapshot; + private final Integer jobPriority; + + AutoIngestJobWrapper(AutoIngestJob job) { + autoIngestJob = job; + jobStage = job.getProcessingStage(); + jobSnapshot = job.getIngestJobSnapshots(); + jobPriority = job.getPriority(); + } + + AutoIngestJob getJob() { + return autoIngestJob; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof AutoIngestJobWrapper)) { + return false; + } + + if (this == other) { + return true; + } + + AutoIngestJob thisJob = this.autoIngestJob; + AutoIngestJob otherJob = ((AutoIngestJobWrapper) other).autoIngestJob; + + // Only equal if the manifest paths and processing stage details are the same. + return thisJob.getManifest().getFilePath().equals(otherJob.getManifest().getFilePath()) + && jobStage.equals(((AutoIngestJobWrapper) other).jobStage) + && jobSnapshot.equals(((AutoIngestJobWrapper) other).jobSnapshot) + && jobPriority.equals(((AutoIngestJobWrapper) other).jobPriority); + } + + @Override + public int hashCode() { + int hash = 7; + hash = 23 * hash + Objects.hashCode(this.autoIngestJob.getManifest().getFilePath()); + hash = 23 * hash + Objects.hashCode(this.jobStage); + hash = 23 * hash + Objects.hashCode(this.jobSnapshot); + hash = 23 * hash + Objects.hashCode(this.jobPriority); + return hash; + } + + @Override + public int compareTo(AutoIngestJobWrapper o) { + return autoIngestJob.compareTo(o.autoIngestJob); + } + + /** + * The remaining methods simply delegate to the wrapped job. + */ + Manifest getManifest() { + return autoIngestJob.getManifest(); + } + + boolean getErrorsOccurred() { + return autoIngestJob.getErrorsOccurred(); + } + + Date getCompletedDate() { + return autoIngestJob.getCompletedDate(); + } + + AutoIngestJob.StageDetails getProcessingStageDetails() { + return autoIngestJob.getProcessingStageDetails(); + } + + String getProcessingHostName() { + return autoIngestJob.getProcessingHostName(); + } + + Integer getPriority() { + return autoIngestJob.getPriority(); + } + } + /** * A ChildFactory for generating JobNodes. */ - static final class AutoIngestNodeChildren extends ChildFactory { + static final class AutoIngestNodeChildren extends ChildFactory { private final AutoIngestJobStatus autoIngestJobStatus; private AutoIngestMonitor monitor; @@ -104,7 +200,7 @@ final class AutoIngestJobsNode extends AbstractNode { } @Override - protected boolean createKeys(List list) { + protected boolean createKeys(List list) { List jobs; switch (autoIngestJobStatus) { case PENDING_JOB: @@ -121,13 +217,15 @@ final class AutoIngestJobsNode extends AbstractNode { jobs = new ArrayList<>(); } if (jobs != null && jobs.size() > 0) { - list.addAll(jobs); + jobs.forEach(j -> { + list.add(new AutoIngestJobWrapper(j)); + }); } return true; } @Override - protected Node createNodeForKey(AutoIngestJob key) { + protected Node createNodeForKey(AutoIngestJobWrapper key) { return new JobNode(key, autoIngestJobStatus, refreshEventBus); } @@ -179,7 +277,7 @@ final class AutoIngestJobsNode extends AbstractNode { */ static final class JobNode extends AbstractNode { - private final AutoIngestJob autoIngestJob; + private final AutoIngestJobWrapper jobWrapper; private final AutoIngestJobStatus jobStatus; private final RefreshNodeSubscriber refreshNodeSubscriber = new RefreshNodeSubscriber(); @@ -190,12 +288,12 @@ final class AutoIngestJobsNode extends AbstractNode { * @param status - the current status of the AutoIngestJob being * represented */ - JobNode(AutoIngestJob job, AutoIngestJobStatus status, EventBus eventBus) { + JobNode(AutoIngestJobWrapper job, AutoIngestJobStatus status, EventBus eventBus) { super(Children.LEAF); jobStatus = status; - autoIngestJob = job; - setName(autoIngestJob.toString()); //alows job to be uniquely found by name since it will involve a hash of the AutoIngestJob - setDisplayName(autoIngestJob.getManifest().getCaseName()); //displays user friendly case name as name + jobWrapper = job; + setName(jobWrapper.toString()); //alows job to be uniquely found by name since it will involve a hash of the AutoIngestJob + setDisplayName(jobWrapper.getManifest().getCaseName()); //displays user friendly case name as name refreshNodeSubscriber.register(eventBus); } @@ -205,7 +303,7 @@ final class AutoIngestJobsNode extends AbstractNode { * @return autoIngestJob */ AutoIngestJob getAutoIngestJob() { - return autoIngestJob; + return jobWrapper.getJob(); } @Override @@ -220,20 +318,20 @@ final class AutoIngestJobsNode extends AbstractNode { s.put(ss); } ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_caseName_text(), Bundle.AutoIngestJobsNode_caseName_text(), Bundle.AutoIngestJobsNode_caseName_text(), - autoIngestJob.getManifest().getCaseName())); + jobWrapper.getManifest().getCaseName())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_dataSource_text(), Bundle.AutoIngestJobsNode_dataSource_text(), Bundle.AutoIngestJobsNode_dataSource_text(), - autoIngestJob.getManifest().getDataSourcePath().getFileName().toString())); + jobWrapper.getManifest().getDataSourcePath().getFileName().toString())); switch (jobStatus) { case PENDING_JOB: ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), - autoIngestJob.getManifest().getDateFileCreated())); + jobWrapper.getManifest().getDateFileCreated())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_priority_text(), Bundle.AutoIngestJobsNode_priority_text(), Bundle.AutoIngestJobsNode_priority_text(), - autoIngestJob.getPriority() > 0 ? Bundle.AutoIngestJobsNode_prioritized_true() : Bundle.AutoIngestJobsNode_prioritized_false())); + jobWrapper.getPriority() > 0 ? Bundle.AutoIngestJobsNode_prioritized_true() : Bundle.AutoIngestJobsNode_prioritized_false())); break; case RUNNING_JOB: - AutoIngestJob.StageDetails status = autoIngestJob.getProcessingStageDetails(); + AutoIngestJob.StageDetails status = jobWrapper.getProcessingStageDetails(); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_hostName_text(), Bundle.AutoIngestJobsNode_hostName_text(), Bundle.AutoIngestJobsNode_hostName_text(), - autoIngestJob.getProcessingHostName())); + jobWrapper.getProcessingHostName())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_stage_text(), Bundle.AutoIngestJobsNode_stage_text(), Bundle.AutoIngestJobsNode_stage_text(), status.getDescription())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_stageTime_text(), Bundle.AutoIngestJobsNode_stageTime_text(), Bundle.AutoIngestJobsNode_stageTime_text(), @@ -241,11 +339,11 @@ final class AutoIngestJobsNode extends AbstractNode { break; case COMPLETED_JOB: ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), - autoIngestJob.getManifest().getDateFileCreated())); + jobWrapper.getManifest().getDateFileCreated())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_jobCompleted_text(), Bundle.AutoIngestJobsNode_jobCompleted_text(), Bundle.AutoIngestJobsNode_jobCompleted_text(), - autoIngestJob.getCompletedDate())); + jobWrapper.getCompletedDate())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_status_text(), Bundle.AutoIngestJobsNode_status_text(), Bundle.AutoIngestJobsNode_status_text(), - autoIngestJob.getErrorsOccurred() ? StatusIconCellRenderer.Status.WARNING : StatusIconCellRenderer.Status.OK)); + jobWrapper.getErrorsOccurred() ? StatusIconCellRenderer.Status.WARNING : StatusIconCellRenderer.Status.OK)); break; default: } @@ -258,24 +356,24 @@ final class AutoIngestJobsNode extends AbstractNode { if (AutoIngestDashboard.isAdminAutoIngestDashboard()) { switch (jobStatus) { case PENDING_JOB: - actions.add(new PrioritizationAction.PrioritizeJobAction(autoIngestJob)); - actions.add(new PrioritizationAction.PrioritizeCaseAction(autoIngestJob)); - PrioritizationAction.DeprioritizeJobAction deprioritizeJobAction = new PrioritizationAction.DeprioritizeJobAction(autoIngestJob); - deprioritizeJobAction.setEnabled(autoIngestJob.getPriority() > 0); + actions.add(new PrioritizationAction.PrioritizeJobAction(jobWrapper.getJob())); + actions.add(new PrioritizationAction.PrioritizeCaseAction(jobWrapper.getJob())); + PrioritizationAction.DeprioritizeJobAction deprioritizeJobAction = new PrioritizationAction.DeprioritizeJobAction(jobWrapper.getJob()); + deprioritizeJobAction.setEnabled(jobWrapper.getPriority() > 0); actions.add(deprioritizeJobAction); - PrioritizationAction.DeprioritizeCaseAction deprioritizeCaseAction = new PrioritizationAction.DeprioritizeCaseAction(autoIngestJob); - deprioritizeCaseAction.setEnabled(autoIngestJob.getPriority() > 0); + PrioritizationAction.DeprioritizeCaseAction deprioritizeCaseAction = new PrioritizationAction.DeprioritizeCaseAction(jobWrapper.getJob()); + deprioritizeCaseAction.setEnabled(jobWrapper.getPriority() > 0); actions.add(deprioritizeCaseAction); break; case RUNNING_JOB: - actions.add(new AutoIngestAdminActions.ProgressDialogAction(autoIngestJob)); - actions.add(new AutoIngestAdminActions.CancelJobAction(autoIngestJob)); + actions.add(new AutoIngestAdminActions.ProgressDialogAction(jobWrapper.getJob())); + actions.add(new AutoIngestAdminActions.CancelJobAction(jobWrapper.getJob())); // actions.add(new AutoIngestAdminActions.CancelModuleAction()); break; case COMPLETED_JOB: - actions.add(new AutoIngestAdminActions.ReprocessJobAction(autoIngestJob)); - actions.add(new AutoIngestAdminActions.DeleteCaseAction(autoIngestJob)); - actions.add(new AutoIngestAdminActions.ShowCaseLogAction(autoIngestJob)); + actions.add(new AutoIngestAdminActions.ReprocessJobAction(jobWrapper.getJob())); + actions.add(new AutoIngestAdminActions.DeleteCaseAction(jobWrapper.getJob())); + actions.add(new AutoIngestAdminActions.ShowCaseLogAction(jobWrapper.getJob())); break; default: } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index 876a56c47e..b8f596abf7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -423,7 +423,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_DEPRIORITIZED, "")); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_DEPRIORITIZED, "")); }).start(); } } @@ -474,7 +474,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_PRIORITIZED, "")); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_PRIORITIZED, "")); }).start(); } } @@ -529,7 +529,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen final String dataSourceName = jobToDeprioritize.getManifest().getDataSourceFileName(); new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_DEPRIORITIZED, dataSourceName)); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_DEPRIORITIZED, dataSourceName)); }).start(); } @@ -590,7 +590,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen final String dataSourceName = jobToPrioritize.getManifest().getDataSourceFileName(); new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_PRIORITIZED, dataSourceName)); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_PRIORITIZED, dataSourceName)); }).start(); } @@ -748,12 +748,11 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * A task that updates the state maintained by the monitor. - * At present this includes auto ingest job and auto ingest node data. - * The job data is refreshed by querying the coordination service for - * auto ingest manifest nodes. - * The auto ingest node data is refreshed by publishing a message asking - * all nodes to report their state. + * A task that updates the state maintained by the monitor. At present this + * includes auto ingest job and auto ingest node data. The job data is + * refreshed by querying the coordination service for auto ingest manifest + * nodes. The auto ingest node data is refreshed by publishing a message + * asking all nodes to report their state. */ private final class StateRefreshTask implements Runnable { @@ -861,100 +860,100 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } - /** - * Class that represents the state of an AIN for the dashboard. - */ - static final class AutoIngestNodeState { - /** - * The set of AIN states. + * Class that represents the state of an AIN for the dashboard. */ - enum State { - STARTING_UP, - SHUTTING_DOWN, - RUNNING, - PAUSE_REQUESTED, - PAUSED_BY_REQUEST, - PAUSED_DUE_TO_SYSTEM_ERROR, - UNKNOWN - } + static final class AutoIngestNodeState { - private final String nodeName; - private final State nodeState; - private final Instant lastSeenTime; - - AutoIngestNodeState(String name, Event event) { - nodeName = name; - switch (event) { - case STARTING_UP: - nodeState = State.STARTING_UP; - break; - case SHUTTING_DOWN: - nodeState = State.SHUTTING_DOWN; - break; - case RUNNING: - nodeState = State.RUNNING; - break; - case PAUSED_BY_USER_REQUEST: - nodeState = State.PAUSED_BY_REQUEST; - break; - case PAUSED_FOR_SYSTEM_ERROR: - nodeState = State.PAUSED_DUE_TO_SYSTEM_ERROR; - break; - case RESUMED: - nodeState = State.RUNNING; - break; - case PAUSE_REQUESTED: - nodeState = State.PAUSE_REQUESTED; - break; - default: - nodeState = State.UNKNOWN; - break; + /** + * The set of AIN states. + */ + enum State { + STARTING_UP, + SHUTTING_DOWN, + RUNNING, + PAUSE_REQUESTED, + PAUSED_BY_REQUEST, + PAUSED_DUE_TO_SYSTEM_ERROR, + UNKNOWN } - lastSeenTime = Instant.now(); - } - String getName() { - return nodeName; - } + private final String nodeName; + private final State nodeState; + private final Instant lastSeenTime; - State getState() { - return nodeState; - } + AutoIngestNodeState(String name, Event event) { + nodeName = name; + switch (event) { + case STARTING_UP: + nodeState = State.STARTING_UP; + break; + case SHUTTING_DOWN: + nodeState = State.SHUTTING_DOWN; + break; + case RUNNING: + nodeState = State.RUNNING; + break; + case PAUSED_BY_USER_REQUEST: + nodeState = State.PAUSED_BY_REQUEST; + break; + case PAUSED_FOR_SYSTEM_ERROR: + nodeState = State.PAUSED_DUE_TO_SYSTEM_ERROR; + break; + case RESUMED: + nodeState = State.RUNNING; + break; + case PAUSE_REQUESTED: + nodeState = State.PAUSE_REQUESTED; + break; + default: + nodeState = State.UNKNOWN; + break; + } + lastSeenTime = Instant.now(); + } - Instant getLastSeenTime() { - return lastSeenTime; - } - } + String getName() { + return nodeName; + } - /** - * Exception type thrown when there is an error completing an auto ingest - * monitor operation. - */ - static final class AutoIngestMonitorException extends Exception { + State getState() { + return nodeState; + } - private static final long serialVersionUID = 1L; - - /** - * Constructs an instance of the exception type thrown when there is an - * error completing an auto ingest monitor operation. - * - * @param message The exception message. - */ - private AutoIngestMonitorException(String message) { - super(message); + Instant getLastSeenTime() { + return lastSeenTime; + } } /** - * Constructs an instance of the exception type thrown when there is an - * error completing an auto ingest monitor operation. - * - * @param message The exception message. - * @param cause A Throwable cause for the error. + * Exception type thrown when there is an error completing an auto + * ingest monitor operation. */ - private AutoIngestMonitorException(String message, Throwable cause) { - super(message, cause); - } + static final class AutoIngestMonitorException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * Constructs an instance of the exception type thrown when there is + * an error completing an auto ingest monitor operation. + * + * @param message The exception message. + */ + private AutoIngestMonitorException(String message) { + super(message); + } + + /** + * Constructs an instance of the exception type thrown when there is + * an error completing an auto ingest monitor operation. + * + * @param message The exception message. + * @param cause A Throwable cause for the error. + */ + private AutoIngestMonitorException(String message, Throwable cause) { + super(message, cause); + } + + } } -} From 5410db46cf7f8ec6f3e25467f7aaa8714dcf6bd8 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 18 Jul 2018 16:09:24 -0400 Subject: [PATCH 75/96] Fix formatting. --- .../autoingest/AutoIngestMonitor.java | 178 +++++++++--------- 1 file changed, 89 insertions(+), 89 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index b8f596abf7..cfdf6cf973 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -698,8 +698,8 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } // Remove jobs associated with this case from the completed jobs collection. - jobsSnapshot.completedJobs.removeIf((AutoIngestJob completedJob) -> - completedJob.getManifest().getCaseName().equals(caseName)); + jobsSnapshot.completedJobs.removeIf((AutoIngestJob completedJob) + -> completedJob.getManifest().getCaseName().equals(caseName)); // Publish a message to update auto ingest nodes. eventPublisher.publishRemotely(new AutoIngestCaseDeletedEvent(caseName, LOCAL_HOST_NAME, AutoIngestManager.getSystemUserNameProperty())); @@ -860,100 +860,100 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } + /** + * Class that represents the state of an AIN for the dashboard. + */ + static final class AutoIngestNodeState { + /** - * Class that represents the state of an AIN for the dashboard. + * The set of AIN states. */ - static final class AutoIngestNodeState { - - /** - * The set of AIN states. - */ - enum State { - STARTING_UP, - SHUTTING_DOWN, - RUNNING, - PAUSE_REQUESTED, - PAUSED_BY_REQUEST, - PAUSED_DUE_TO_SYSTEM_ERROR, - UNKNOWN - } - - private final String nodeName; - private final State nodeState; - private final Instant lastSeenTime; - - AutoIngestNodeState(String name, Event event) { - nodeName = name; - switch (event) { - case STARTING_UP: - nodeState = State.STARTING_UP; - break; - case SHUTTING_DOWN: - nodeState = State.SHUTTING_DOWN; - break; - case RUNNING: - nodeState = State.RUNNING; - break; - case PAUSED_BY_USER_REQUEST: - nodeState = State.PAUSED_BY_REQUEST; - break; - case PAUSED_FOR_SYSTEM_ERROR: - nodeState = State.PAUSED_DUE_TO_SYSTEM_ERROR; - break; - case RESUMED: - nodeState = State.RUNNING; - break; - case PAUSE_REQUESTED: - nodeState = State.PAUSE_REQUESTED; - break; - default: - nodeState = State.UNKNOWN; - break; - } - lastSeenTime = Instant.now(); - } - - String getName() { - return nodeName; - } - - State getState() { - return nodeState; - } - - Instant getLastSeenTime() { - return lastSeenTime; - } + enum State { + STARTING_UP, + SHUTTING_DOWN, + RUNNING, + PAUSE_REQUESTED, + PAUSED_BY_REQUEST, + PAUSED_DUE_TO_SYSTEM_ERROR, + UNKNOWN } - /** - * Exception type thrown when there is an error completing an auto - * ingest monitor operation. - */ - static final class AutoIngestMonitorException extends Exception { + private final String nodeName; + private final State nodeState; + private final Instant lastSeenTime; - private static final long serialVersionUID = 1L; - - /** - * Constructs an instance of the exception type thrown when there is - * an error completing an auto ingest monitor operation. - * - * @param message The exception message. - */ - private AutoIngestMonitorException(String message) { - super(message); + AutoIngestNodeState(String name, Event event) { + nodeName = name; + switch (event) { + case STARTING_UP: + nodeState = State.STARTING_UP; + break; + case SHUTTING_DOWN: + nodeState = State.SHUTTING_DOWN; + break; + case RUNNING: + nodeState = State.RUNNING; + break; + case PAUSED_BY_USER_REQUEST: + nodeState = State.PAUSED_BY_REQUEST; + break; + case PAUSED_FOR_SYSTEM_ERROR: + nodeState = State.PAUSED_DUE_TO_SYSTEM_ERROR; + break; + case RESUMED: + nodeState = State.RUNNING; + break; + case PAUSE_REQUESTED: + nodeState = State.PAUSE_REQUESTED; + break; + default: + nodeState = State.UNKNOWN; + break; } + lastSeenTime = Instant.now(); + } - /** - * Constructs an instance of the exception type thrown when there is - * an error completing an auto ingest monitor operation. - * - * @param message The exception message. - * @param cause A Throwable cause for the error. - */ - private AutoIngestMonitorException(String message, Throwable cause) { - super(message, cause); - } + String getName() { + return nodeName; + } + State getState() { + return nodeState; + } + + Instant getLastSeenTime() { + return lastSeenTime; } } + + /** + * Exception type thrown when there is an error completing an auto ingest + * monitor operation. + */ + static final class AutoIngestMonitorException extends Exception { + + private static final long serialVersionUID = 1L; + + /** + * Constructs an instance of the exception type thrown when there is an + * error completing an auto ingest monitor operation. + * + * @param message The exception message. + */ + private AutoIngestMonitorException(String message) { + super(message); + } + + /** + * Constructs an instance of the exception type thrown when there is an + * error completing an auto ingest monitor operation. + * + * @param message The exception message. + * @param cause A Throwable cause for the error. + */ + private AutoIngestMonitorException(String message, Throwable cause) { + super(message, cause); + } + + } +} From e5a8ae21f557ec6900d2cf4279b0ef21e7e40936 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 18 Jul 2018 16:54:36 -0400 Subject: [PATCH 76/96] Reusing GeneralFilters from Image DSP. --- .../autopsy/casemodule/Bundle.properties | 1 - .../autopsy/casemodule/Bundle_ja.properties | 1 - .../autopsy/casemodule/ImageDSProcessor.java | 18 ++++++++++ .../casemodule/MissingImageDialog.java | 35 +++++++------------ 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties index 9c04508c1f..9494b3288a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties @@ -114,7 +114,6 @@ LocalFilesDSProcessor.toString.text=Logical Files LocalFilesPanel.contentType.text=LOCAL LocalFilesPanel.moduleErr=Module Error LocalFilesPanel.moduleErr.msg=A module caused an error listening to LocalFilesPanel updates. See log to determine which module. Some data could be incomplete. -MissingImageDialog.allDesc.text=All Supported Types MissingImageDialog.display.title=Search for Missing Image MissingImageDialog.confDlg.noFileSel.msg=No image file has been selected. Are you sure you\nwould like to exit without finding the image? MissingImageDialog.confDlg.noFileSel.title=Missing Image diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties index 8e405adec4..90849f4639 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties @@ -101,7 +101,6 @@ LocalFilesDSProcessor.toString.text=\u30ed\u30b8\u30ab\u30eb\u30d5\u30a1\u30a4\u LocalFilesPanel.contentType.text=\u30ed\u30fc\u30ab\u30eb LocalFilesPanel.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc LocalFilesPanel.moduleErr.msg=LocalFilesPanel\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3092\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 -MissingImageDialog.allDesc.text=\u30b5\u30dd\u30fc\u30c8\u3055\u308c\u3066\u3044\u308b\u5168\u3066\u306e\u30bf\u30a4\u30d7 MissingImageDialog.display.title=\u6b20\u843d\u30a4\u30e1\u30fc\u30b8\u3092\u691c\u7d22 MissingImageDialog.confDlg.noFileSel.msg=\u30a4\u30e1\u30fc\u30b8\u30d5\u30a1\u30a4\u30eb\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u30a4\u30e1\u30fc\u30b8\u3092\u898b\u3064\u3051\u308b\n\u524d\u306b\u672c\u5f53\u306b\u7d42\u4e86\u3057\u307e\u3059\u304b\uff1f MissingImageDialog.confDlg.noFileSel.title=\u6b20\u843d\u30a4\u30e1\u30fc\u30b8 diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java index 1e091db81c..fccf740d2e 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java @@ -89,6 +89,24 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour public ImageDSProcessor() { configPanel = ImageFilePanel.createInstance(ImageDSProcessor.class.getName(), filtersList); } + + /** + * Get the list of file extensions supported by this DSP. + * + * @return A list of all supported file extensions. + */ + static List getFileExtensions() { + return allExt; + } + + /** + * Get the list of file filters supported by this DSP. + * + * @return A list of all supported file filters. + */ + static List getFileFiltersList() { + return filtersList; + } /** * Gets a string that describes the type of data sources this processor is diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java b/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java index 78cef08b25..894603fe02 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/MissingImageDialog.java @@ -28,6 +28,7 @@ import java.io.File; import javax.swing.JFileChooser; import javax.swing.JFrame; import javax.swing.JOptionPane; +import javax.swing.filechooser.FileFilter; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.coreutils.DriveUtils; @@ -44,19 +45,8 @@ class MissingImageDialog extends javax.swing.JDialog { private static final Logger logger = Logger.getLogger(MissingImageDialog.class.getName()); long obj_id; SleuthkitCase db; - static final GeneralFilter rawFilter = new GeneralFilter(GeneralFilter.RAW_IMAGE_EXTS, GeneralFilter.RAW_IMAGE_DESC); - static final GeneralFilter encaseFilter = new GeneralFilter(GeneralFilter.ENCASE_IMAGE_EXTS, GeneralFilter.ENCASE_IMAGE_DESC); - static final GeneralFilter virtualMachineFilter = new GeneralFilter(GeneralFilter.VIRTUAL_MACHINE_EXTS, GeneralFilter.VIRTUAL_MACHINE_DESC); - static final List allExt = new ArrayList(); - static { - allExt.addAll(GeneralFilter.RAW_IMAGE_EXTS); - allExt.addAll(GeneralFilter.ENCASE_IMAGE_EXTS); - allExt.addAll(GeneralFilter.VIRTUAL_MACHINE_EXTS); - } - static final String allDesc = NbBundle.getMessage(MissingImageDialog.class, "MissingImageDialog.allDesc.text"); - static final GeneralFilter allFilter = new GeneralFilter(allExt, allDesc); - private final JFileChooser fc = new JFileChooser(); + private final JFileChooser fileChooser = new JFileChooser(); /** * Instantiate a MissingImageDialog. @@ -70,14 +60,15 @@ class MissingImageDialog extends javax.swing.JDialog { this.db = db; initComponents(); - fc.setDragEnabled(false); - fc.setFileSelectionMode(JFileChooser.FILES_ONLY); - fc.setMultiSelectionEnabled(false); + fileChooser.setDragEnabled(false); + fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + fileChooser.setMultiSelectionEnabled(false); - fc.addChoosableFileFilter(rawFilter); - fc.addChoosableFileFilter(encaseFilter); - fc.addChoosableFileFilter(virtualMachineFilter); - fc.setFileFilter(allFilter); + List fileFiltersList = ImageDSProcessor.getFileFiltersList(); + for (FileFilter fileFilter : fileFiltersList) { + fileChooser.addChoosableFileFilter(fileFilter); + } + fileChooser.setFileFilter(fileFiltersList.get(0)); selectButton.setEnabled(false); } @@ -289,12 +280,12 @@ class MissingImageDialog extends javax.swing.JDialog { // set the current directory of the FileChooser if the ImagePath Field is valid File currentDir = new File(oldText); if (currentDir.exists()) { - fc.setCurrentDirectory(currentDir); + fileChooser.setCurrentDirectory(currentDir); } - int retval = fc.showOpenDialog(this); + int retval = fileChooser.showOpenDialog(this); if (retval == JFileChooser.APPROVE_OPTION) { - String path = fc.getSelectedFile().getPath(); + String path = fileChooser.getSelectedFile().getPath(); pathNameTextField.setText(path); } //pcs.firePropertyChange(DataSourceProcessor.DSP_PANEL_EVENT.FOCUS_NEXT.toString(), false, true); From b3fafcf06ad9e28901de4cf19decfd9a97f90516 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 18 Jul 2018 16:57:11 -0400 Subject: [PATCH 77/96] Removed unused method. --- .../sleuthkit/autopsy/casemodule/ImageDSProcessor.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java index fccf740d2e..0fb98ea8b8 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java @@ -90,15 +90,6 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour configPanel = ImageFilePanel.createInstance(ImageDSProcessor.class.getName(), filtersList); } - /** - * Get the list of file extensions supported by this DSP. - * - * @return A list of all supported file extensions. - */ - static List getFileExtensions() { - return allExt; - } - /** * Get the list of file filters supported by this DSP. * From a2ad970a7c853741883aed6963cd2fb9d4c79e36 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 19 Jul 2018 08:47:09 -0400 Subject: [PATCH 78/96] Move call to isOpened onto the EDT --- .../autopsy/directorytree/DirectoryTreeTopComponent.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 95ba2f110c..c84d0ee1f2 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -735,9 +735,11 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * responsible for opening core windows. Consider moving * this elsewhere. */ - if (!this.isOpened()) { - SwingUtilities.invokeLater(CoreComponentControl::openCoreWindows); - } + SwingUtilities.invokeLater(() -> { + if (! DirectoryTreeTopComponent.this.isOpened()) { + CoreComponentControl.openCoreWindows(); + } + }); } catch (NoCurrentCaseException notUsed) { /** * Case is closed, do nothing. From f1e77aac8991dc67fe2cecf5265fbd4d41f7e71d Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 19 Jul 2018 09:09:36 -0400 Subject: [PATCH 79/96] set VHD replace option by default. --- Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java index cb1ec5ddfd..c5c8fb32fb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java @@ -257,6 +257,7 @@ final class LocalDiskPanel extends JPanel { pathTextField.setEnabled(copyImageCheckbox.isSelected()); browseButton.setEnabled(copyImageCheckbox.isSelected()); changeDatabasePathCheckbox.setEnabled(copyImageCheckbox.isSelected()); + changeDatabasePathCheckbox.setSelected(copyImageCheckbox.isSelected()); fireUpdateEvent(); }//GEN-LAST:event_copyImageCheckboxActionPerformed From aba1489aeebc07ba3614022fee23bb7177d1d7b9 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 17 Jul 2018 10:43:33 -0400 Subject: [PATCH 80/96] 4037 Close InputStream in object detection module which was left open --- .../ObjectDetectectionFileIngestModule.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index d6bef92ab8..86a35dce23 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -101,7 +101,7 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter public ProcessResult process(AbstractFile file) { if (!classifiers.isEmpty() && ImageUtils.isImageThumbnailSupported(file)) { //Any image we can create a thumbnail for is one we should apply the classifiers to - InputStream inputStream = new ReadContentInputStream(file); + InputStream inputStream = new ReadContentInputStream(file); byte[] imageInMemory; try { imageInMemory = IOUtils.toByteArray(inputStream); @@ -109,6 +109,15 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); return IngestModule.ProcessResult.ERROR; } + finally { + try { + inputStream.close(); + } catch (IOException ex) { + logger.log(Level.SEVERE, "Unable to close input stream after attempting to create byte array for " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); + return IngestModule.ProcessResult.ERROR; + } + } + Mat originalImage; try { originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); From 04068dd4928a47974fd1407105dc629c2fdd6350 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 17 Jul 2018 11:10:04 -0400 Subject: [PATCH 81/96] 4037 entirely remove unnecessary input stream in Object Detection --- .../ObjectDetectectionFileIngestModule.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index 86a35dce23..0e9155617c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -33,6 +33,7 @@ import org.opencv.core.MatOfByte; import org.opencv.core.MatOfRect; import org.opencv.highgui.Highgui; import org.opencv.objdetect.CascadeClassifier; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -100,23 +101,15 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter @Override public ProcessResult process(AbstractFile file) { if (!classifiers.isEmpty() && ImageUtils.isImageThumbnailSupported(file)) { - //Any image we can create a thumbnail for is one we should apply the classifiers to - InputStream inputStream = new ReadContentInputStream(file); - byte[] imageInMemory; + //Any image we can create a thumbnail for is one we should apply the classifiers to + byte[] imageInMemory = new byte[(int)file.getSize()]; + try { - imageInMemory = IOUtils.toByteArray(inputStream); - } catch (IOException ex) { + file.read(imageInMemory, 0, file.getSize()); + } catch (TskCoreException ex) { logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); return IngestModule.ProcessResult.ERROR; } - finally { - try { - inputStream.close(); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Unable to close input stream after attempting to create byte array for " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); - return IngestModule.ProcessResult.ERROR; - } - } Mat originalImage; try { From 8d546a217e6011be251baf9122409a24e2cb059e Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 17 Jul 2018 11:14:45 -0400 Subject: [PATCH 82/96] remove unnecessary imports from Object detection --- .../objectdetection/ObjectDetectectionFileIngestModule.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index 0e9155617c..398bed8010 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -19,21 +19,17 @@ package org.sleuthkit.autopsy.experimental.objectdetection; import java.io.File; -import java.io.IOException; -import java.io.InputStream; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.logging.Level; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; import org.opencv.core.CvException; import org.opencv.core.Mat; import org.opencv.core.MatOfByte; import org.opencv.core.MatOfRect; import org.opencv.highgui.Highgui; import org.opencv.objdetect.CascadeClassifier; -import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -53,7 +49,6 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_OBJECT_DETECTED; import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.TskCoreException; /** From fad0f574063312f3895e0bfcad2277267b4fc148 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 17 Jul 2018 11:27:53 -0400 Subject: [PATCH 83/96] 4037 introduce max file size to object detection module (100 mb) --- .../ObjectDetectectionFileIngestModule.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index 398bed8010..a6bf2a2ee6 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -57,6 +57,7 @@ import org.sleuthkit.datamodel.TskCoreException; public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter { private final static Logger logger = Logger.getLogger(ObjectDetectectionFileIngestModule.class.getName()); + private final static int MAX_FILE_SIZE = 100000000; //Max size of pictures to perform object detection on private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); private long jobId; private Map classifiers; @@ -97,6 +98,14 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter public ProcessResult process(AbstractFile file) { if (!classifiers.isEmpty() && ImageUtils.isImageThumbnailSupported(file)) { //Any image we can create a thumbnail for is one we should apply the classifiers to + + if (file.getSize() > MAX_FILE_SIZE){ + //prevent it from allocating gigabytes of memory for extremely large files + logger.log(Level.INFO, "Encountered file "+ file.getParentPath() + file.getName() + " with object id of " + + file.getId() + " which exceeds max file size of " + MAX_FILE_SIZE + " bytes, with a size of " + file.getSize()); + return IngestModule.ProcessResult.OK; + } + byte[] imageInMemory = new byte[(int)file.getSize()]; try { From fb28c62d6059d8d13f314ac803291ae985d837fc Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 18 Jul 2018 11:54:16 -0400 Subject: [PATCH 84/96] 4037 add explicit calls to release opencv objects to avoid reliance on finalize method --- .../ObjectDetectectionFileIngestModule.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index a6bf2a2ee6..003a35a46d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -98,15 +98,15 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter public ProcessResult process(AbstractFile file) { if (!classifiers.isEmpty() && ImageUtils.isImageThumbnailSupported(file)) { //Any image we can create a thumbnail for is one we should apply the classifiers to - - if (file.getSize() > MAX_FILE_SIZE){ - //prevent it from allocating gigabytes of memory for extremely large files - logger.log(Level.INFO, "Encountered file "+ file.getParentPath() + file.getName() + " with object id of " + - file.getId() + " which exceeds max file size of " + MAX_FILE_SIZE + " bytes, with a size of " + file.getSize()); + + if (file.getSize() > MAX_FILE_SIZE) { + //prevent it from allocating gigabytes of memory for extremely large files + logger.log(Level.INFO, "Encountered file " + file.getParentPath() + file.getName() + " with object id of " + + file.getId() + " which exceeds max file size of " + MAX_FILE_SIZE + " bytes, with a size of " + file.getSize()); return IngestModule.ProcessResult.OK; } - - byte[] imageInMemory = new byte[(int)file.getSize()]; + + byte[] imageInMemory = new byte[(int) file.getSize()]; try { file.read(imageInMemory, 0, file.getSize()); @@ -114,7 +114,7 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); return IngestModule.ProcessResult.ERROR; } - + Mat originalImage; try { originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); @@ -127,7 +127,6 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), unexpectedException); return IngestModule.ProcessResult.ERROR; } - MatOfRect detectionRectangles = new MatOfRect(); //the rectangles which reprent the coordinates on the image for where objects were detected for (String classifierKey : classifiers.keySet()) { //apply each classifier to the file @@ -136,9 +135,9 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter } catch (CvException ignored) { //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable continue; - } catch (Exception unexpectedException) { + } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " with object id of " + file.getId() +" while trying to apply classifier " + classifierKey, unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " with object id of " + file.getId() + " while trying to apply classifier " + classifierKey, unexpectedException); continue; } @@ -169,10 +168,14 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter } catch (TskCoreException ex) { logger.log(Level.SEVERE, String.format("Failed to create blackboard artifact for '%s'.", file.getParentPath() + file.getName()), ex); //NON-NLS + detectionRectangles.release(); + originalImage.release(); return IngestModule.ProcessResult.ERROR; } } } + detectionRectangles.release(); + originalImage.release(); } return IngestModule.ProcessResult.OK; From e970e8b71c76305efc1294619eac353d10547798 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 19 Jul 2018 10:59:30 -0400 Subject: [PATCH 85/96] Make sure setupTable is called from the EDT --- .../autopsy/corecomponents/DataResultViewerTable.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 6ccf1a3c46..7f9049e414 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -141,7 +141,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { * Configure the child OutlineView (explorer view) component. */ outlineView.setAllowedDragActions(DnDConstants.ACTION_NONE); - + outline = outlineView.getOutline(); outline.setRowSelectionAllowed(true); outline.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); @@ -252,6 +252,10 @@ public class DataResultViewerTable extends AbstractDataResultViewer { * persisted column order, sorting and visibility is used. */ private void setupTable() { + if (! SwingUtilities.isEventDispatchThread()) { + LOGGER.log(Level.SEVERE, "Attempting to run setupTable() from non-EDT thread"); + return; + } /* * Since we are modifying the columns, we don't want to listen to * added/removed events as un-hide/hide, until the table setup is done. @@ -281,7 +285,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { * let the table resize itself. */ outline.setAutoResizeMode((props.isEmpty()) ? JTable.AUTO_RESIZE_ALL_COLUMNS : JTable.AUTO_RESIZE_OFF); - + assignColumns(props); // assign columns to match the properties if (firstProp != null) { ((DefaultOutlineModel) outline.getOutlineModel()).setNodesColumnLabel(firstProp.getDisplayName()); From 771f8489a8b940dedeffd1a798dc4730644d6047 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 19 Jul 2018 11:03:41 -0400 Subject: [PATCH 86/96] Moved EDT check up to setNode() --- .../autopsy/corecomponents/DataResultViewerTable.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 7f9049e414..80d42614c1 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -208,6 +208,11 @@ public class DataResultViewerTable extends AbstractDataResultViewer { @Override @ThreadConfined(type = ThreadConfined.ThreadType.AWT) public void setNode(Node rootNode) { + if (! SwingUtilities.isEventDispatchThread()) { + LOGGER.log(Level.SEVERE, "Attempting to run setNode() from non-EDT thread"); + return; + } + /* * The quick filter must be reset because when determining column width, * ETable.getRowCount is called, and the documentation states that quick @@ -252,10 +257,6 @@ public class DataResultViewerTable extends AbstractDataResultViewer { * persisted column order, sorting and visibility is used. */ private void setupTable() { - if (! SwingUtilities.isEventDispatchThread()) { - LOGGER.log(Level.SEVERE, "Attempting to run setupTable() from non-EDT thread"); - return; - } /* * Since we are modifying the columns, we don't want to listen to * added/removed events as un-hide/hide, until the table setup is done. From 5ab3050362d68c64e8b3f8757d8ba6d39f89b26e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 19 Jul 2018 11:15:33 -0400 Subject: [PATCH 87/96] Fixed bugs around checkboxes. --- .../autopsy/report/ReportVisualPanel2.form | 3 ++ .../autopsy/report/ReportVisualPanel2.java | 50 ++++++++++++++----- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.form b/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.form index e1f23ebffc..a67060d6f3 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.form +++ b/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.form @@ -97,6 +97,9 @@ + + + diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.java b/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.java index 82de40aac9..03f3bda809 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportVisualPanel2.java @@ -109,7 +109,7 @@ final class ReportVisualPanel2 extends JPanel { } for (TagName tagName : tagNamesInUse) { - String notableString = tagName.getKnownStatus() == TskData.FileKnown.BAD ? TagsManager.getNotableTagLabel() : ""; + String notableString = tagName.getKnownStatus() == TskData.FileKnown.BAD ? TagsManager.getNotableTagLabel() : ""; tagStates.put(tagName.getDisplayName() + notableString, Boolean.FALSE); } tags.addAll(tagStates.keySet()); @@ -124,7 +124,9 @@ final class ReportVisualPanel2 extends JPanel { tagsList.addMouseListener(new MouseAdapter() { @Override public void mousePressed(MouseEvent evt) { - + if (!taggedResultsRadioButton.isSelected()) { + return; + } int index = tagsList.locationToIndex(evt.getPoint()); if (index < tagsModel.getSize() && index >= 0) { String value = tagsModel.getElementAt(index); @@ -184,16 +186,26 @@ final class ReportVisualPanel2 extends JPanel { return tagStates; } + /** + * Are any tags selected? + * + * @return True if any tags are selected; otherwise false. + */ private boolean areTagsSelected() { boolean result = false; for (Entry entry : tagStates.entrySet()) { if (entry.getValue()) { result = true; + break; } } return result; } + /** + * Set the Finish button as either enabled or disabled depending on the UI + * component selections. + */ private void updateFinishButton() { if (taggedResultsRadioButton.isSelected()) { wizPanel.setFinish(areTagsSelected()); @@ -209,6 +221,19 @@ final class ReportVisualPanel2 extends JPanel { return taggedResultsRadioButton.isSelected(); } + /** + * Set all tagged results as either selected or unselected. + * + * @param selected Should all tagged results be selected? + */ + void setAllTaggedResultsSelected(boolean selected) { + for (String tag : tags) { + tagStates.put(tag, (selected ? Boolean.TRUE : Boolean.FALSE)); + } + tagsList.repaint(); + wizPanel.setFinish(selected); + } + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -234,6 +259,11 @@ final class ReportVisualPanel2 extends JPanel { optionsButtonGroup.add(allResultsRadioButton); org.openide.awt.Mnemonics.setLocalizedText(allResultsRadioButton, org.openide.util.NbBundle.getMessage(ReportVisualPanel2.class, "ReportVisualPanel2.allResultsRadioButton.text")); // NOI18N + allResultsRadioButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + allResultsRadioButtonActionPerformed(evt); + } + }); org.openide.awt.Mnemonics.setLocalizedText(dataLabel, org.openide.util.NbBundle.getMessage(ReportVisualPanel2.class, "ReportVisualPanel2.dataLabel.text")); // NOI18N @@ -312,25 +342,21 @@ final class ReportVisualPanel2 extends JPanel { }// //GEN-END:initComponents private void selectAllButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_selectAllButtonActionPerformed - for (String tag : tags) { - tagStates.put(tag, Boolean.TRUE); - } - tagsList.repaint(); - wizPanel.setFinish(true); + setAllTaggedResultsSelected(true); }//GEN-LAST:event_selectAllButtonActionPerformed private void deselectAllButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deselectAllButtonActionPerformed - for (String tag : tags) { - tagStates.put(tag, Boolean.FALSE); - } - tagsList.repaint(); - wizPanel.setFinish(false); + setAllTaggedResultsSelected(false); }//GEN-LAST:event_deselectAllButtonActionPerformed private void advancedButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_advancedButtonActionPerformed artifactStates = dialog.display(); }//GEN-LAST:event_advancedButtonActionPerformed + private void allResultsRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_allResultsRadioButtonActionPerformed + setAllTaggedResultsSelected(false); + }//GEN-LAST:event_allResultsRadioButtonActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton advancedButton; private javax.swing.JRadioButton allResultsRadioButton; From e98e0db15b9139a50171a52a5fca4bc964a3edb4 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 19 Jul 2018 13:41:31 -0400 Subject: [PATCH 88/96] Revert to old method of calling setupTabs --- .../corecomponents/DataResultPanel.java | 34 +++++-------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java index ced154edf7..9d0a83c2ad 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java @@ -26,7 +26,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import javax.swing.JTabbedPane; -import javax.swing.SwingWorker; +import javax.swing.SwingUtilities; import javax.swing.event.ChangeEvent; import javax.swing.event.ChangeListener; import org.openide.explorer.ExplorerManager; @@ -579,29 +579,6 @@ public class DataResultPanel extends javax.swing.JPanel implements DataResult, C } } - /** - * Worker for RootNodeListener childrenAdded. - */ - class SetupTabsChildrenWorker extends SwingWorker { - - private final Node childNode; - - SetupTabsChildrenWorker(Node aChildNode) { - childNode = aChildNode; - } - - @Override - protected Void doInBackground() throws Exception { - setupTabs(childNode); - return null; - } - - @Override - protected void done() { - setupTabs(childNode); - } - } - /** * Responds to changes in the root node due to asynchronous child node * creation. @@ -628,8 +605,13 @@ public class DataResultPanel extends javax.swing.JPanel implements DataResult, C */ if (waitingForData && containsReal(delta)) { waitingForData = false; - Node childNode = nme.getNode(); - new SetupTabsChildrenWorker(childNode).execute(); + if (SwingUtilities.isEventDispatchThread()) { + setupTabs(nme.getNode()); + } else { + SwingUtilities.invokeLater(() -> { + setupTabs(nme.getNode()); + }); + } } } From 8764aeba87ee48f2ba9948c76ebc2126fcf98b5d Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 19 Jul 2018 14:02:04 -0400 Subject: [PATCH 89/96] Update TSKVersion.xml --- TSKVersion.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TSKVersion.xml b/TSKVersion.xml index 57096d8d35..7350f25b66 100644 --- a/TSKVersion.xml +++ b/TSKVersion.xml @@ -1,3 +1,3 @@ - + From d70802cb629b851fee1fc16ed7b6866644d56e37 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 20 Jul 2018 14:55:59 +0200 Subject: [PATCH 90/96] add tooltip to timeline listview description column --- .../sleuthkit/autopsy/timeline/ui/listvew/ListTimeline.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/listvew/ListTimeline.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/listvew/ListTimeline.java index 7515891356..6bc26f8d7a 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/listvew/ListTimeline.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/listvew/ListTimeline.java @@ -567,8 +567,11 @@ class ListTimeline extends BorderPane { super.updateItem(item, empty); if (empty || item == null) { setText(null); + setTooltip(null); } else { - setText(textSupplier.apply(getEvent())); + String text = textSupplier.apply(getEvent()); + setText(text); + setTooltip(new Tooltip(text)); } } } From ffd45d3bf5d96f49d19766b05522f86087146544 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 20 Jul 2018 10:27:42 -0400 Subject: [PATCH 91/96] update version --- Core/nbproject/project.properties | 3 +-- Core/nbproject/project.xml | 4 ++-- unix_setup.sh | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index b596512161..9ff74aca14 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -17,7 +17,6 @@ file.reference.sevenzipjbinding-AllPlatforms.jar=release/modules/ext/sevenzipjbi file.reference.sevenzipjbinding.jar=release/modules/ext/sevenzipjbinding.jar file.reference.sqlite-jdbc-3.8.11.jar=release/modules/ext/sqlite-jdbc-3.8.11.jar file.reference.StixLib.jar=release/modules/ext/StixLib.jar -file.reference.sleuthkit-postgresql-4.6.1.jar=release/modules/ext/sleuthkit-postgresql-4.6.1.jar file.reference.bcprov-jdk15on-1.54.jar=release/modules/ext/bcprov-jdk15on-1.54.jar file.reference.jackcess-2.1.8.jar=release/modules/ext/jackcess-2.1.8.jar file.reference.jackcess-encrypt-2.1.2.jar=release/modules/ext/jackcess-encrypt-2.1.2.jar @@ -30,7 +29,7 @@ file.reference.cxf-rt-transports-http-3.0.16.jar=release/modules/ext/cxf-rt-tran file.reference.fontbox-2.0.8.jar=release/modules/ext/fontbox-2.0.8.jar file.reference.pdfbox-2.0.8.jar=release/modules/ext/pdfbox-2.0.8.jar file.reference.pdfbox-tools-2.0.8.jar=release/modules/ext/pdfbox-tools-2.0.8.jar -file.reference.sleuthkit-postgresql-4.6.1.jar=release/modules/ext/sleuthkit-postgresql-4.6.1.jar +file.reference.sleuthkit-postgresql-4.6.2.jar=release/modules/ext/sleuthkit-postgresql-4.6.2.jar file.reference.tika-core-1.17.jar=release/modules/ext/tika-core-1.17.jar file.reference.tika-parsers-1.17.jar=release/modules/ext/tika-parsers-1.17.jar file.reference.curator-client-2.8.0.jar=release/modules/ext/curator-client-2.8.0.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 57cbc58c25..46ad7c5426 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -412,8 +412,8 @@ release/modules/ext/metadata-extractor-2.10.1.jar - ext/sleuthkit-postgresql-4.6.1.jar - release/modules/ext/sleuthkit-postgresql-4.6.1.jar + ext/sleuthkit-postgresql-4.6.2.jar + release/modules/ext/sleuthkit-postgresql-4.6.2.jar ext/tika-core-1.17.jar diff --git a/unix_setup.sh b/unix_setup.sh index 970c35a5c2..1cf9b65888 100755 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -2,7 +2,7 @@ # Verifies programs are installed and copies native code into the Autopsy folder structure -TSK_VERSION=4.6.1 +TSK_VERSION=4.6.2 # Verify PhotoRec was installed photorec_filepath=/usr/bin/photorec From 821eff842d91b538e291705933e83c24f60ed78a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 26 Jul 2018 15:15:43 -0400 Subject: [PATCH 92/96] Make new public AbstractAbstractFileNode protected --- .../autopsy/commonfilesearch/FileInstanceNode.java | 2 +- .../datamodel/AbstractAbstractFileNode.java | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNode.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNode.java index 07c87035e1..899aa5abbc 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNode.java @@ -77,7 +77,7 @@ public class FileInstanceNode extends FileNode { sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), NO_DESCR, this.getContent().getName())); sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), NO_DESCR, this.getContent().getParentPath())); - sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), NO_DESCR, getHashSetHitsForFile(this.getContent()))); + sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), NO_DESCR, getHashSetHitsCsvList(this.getContent()))); sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, this.getDataSource())); sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), NO_DESCR, StringUtils.defaultString(this.getContent().getMIMEType()))); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 28329f17c5..1a52a61bb2 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -251,7 +251,7 @@ public abstract class AbstractAbstractFileNode extends A map.put(TYPE_DIR.toString(), content.getDirType().getLabel()); map.put(TYPE_META.toString(), content.getMetaType().toString()); map.put(KNOWN.toString(), content.getKnown().getName()); - map.put(HASHSETS.toString(), getHashSetHitsForFile(content)); + map.put(HASHSETS.toString(), getHashSetHitsCsvList(content)); map.put(MD5HASH.toString(), StringUtils.defaultString(content.getMd5Hash())); map.put(ObjectID.toString(), content.getId()); map.put(MIMETYPE.toString(), StringUtils.defaultString(content.getMIMEType())); @@ -263,7 +263,7 @@ public abstract class AbstractAbstractFileNode extends A * to their sheets. * * @param sheetSet the modifiable Sheet.Set returned by - * Sheet.get(Sheet.PROPERTIES) + * Sheet.get(Sheet.PROPERTIES) */ @NbBundle.Messages("AbstractAbstractFileNode.tagsProperty.displayName=Tags") protected void addTagProperty(Sheet.Set sheetSet) { @@ -301,7 +301,15 @@ public abstract class AbstractAbstractFileNode extends A } } - public static String getHashSetHitsForFile(AbstractFile file) { + /** + * Gets a comma-separated values list of the names of the hash sets + * currently identified as including a given file. + * + * @param file The file. + * + * @return The CSV list of hash set names. + */ + protected static String getHashSetHitsCsvList(AbstractFile file) { try { return StringUtils.join(file.getHashSetNames(), ", "); } catch (TskCoreException tskCoreException) { From 42835825a9533da9ead8ecf85ca04db981e76b4b Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 27 Jul 2018 18:06:12 -0400 Subject: [PATCH 93/96] AutopsyTreeChildrenFactory => final AutopsyTreeChildFactory --- ...ildrenFactory.java => AutopsyTreeChildFactory.java} | 4 ++-- .../directorytree/DirectoryTreeTopComponent.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) rename Core/src/org/sleuthkit/autopsy/datamodel/{AutopsyTreeChildrenFactory.java => AutopsyTreeChildFactory.java} (97%) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildrenFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java similarity index 97% rename from Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildrenFactory.java rename to Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java index 9573347eb9..70f0ca3ce6 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildrenFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java @@ -41,9 +41,9 @@ import org.sleuthkit.datamodel.TskCoreException; * Child factory to create the top level children of the autopsy tree * */ -public class AutopsyTreeChildrenFactory extends ChildFactory.Detachable { +public final class AutopsyTreeChildFactory extends ChildFactory.Detachable { - private static final Logger logger = Logger.getLogger(AutopsyTreeChildrenFactory.class.getName()); + private static final Logger logger = Logger.getLogger(AutopsyTreeChildFactory.class.getName()); /** * Listener for handling DATA_SOURCE_ADDED events. diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index c84d0ee1f2..fb9ded33e2 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -80,7 +80,7 @@ import org.sleuthkit.autopsy.datamodel.FileTypesByMimeType; import org.sleuthkit.autopsy.datamodel.InterestingHits; import org.sleuthkit.autopsy.datamodel.KeywordHits; import org.sleuthkit.autopsy.datamodel.ResultsNode; -import org.sleuthkit.autopsy.datamodel.AutopsyTreeChildrenFactory; +import org.sleuthkit.autopsy.datamodel.AutopsyTreeChildFactory; import org.sleuthkit.autopsy.datamodel.ViewsNode; import org.sleuthkit.autopsy.datamodel.accounts.Accounts; import org.sleuthkit.autopsy.datamodel.accounts.BINRange; @@ -108,7 +108,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private final LinkedList forwardList; private static final String PREFERRED_ID = "DirectoryTreeTopComponent"; //NON-NLS private static final Logger LOGGER = Logger.getLogger(DirectoryTreeTopComponent.class.getName()); - private AutopsyTreeChildrenFactory autopsyTreeChildrenFactory; + private AutopsyTreeChildFactory autopsyTreeChildFactory; private Children autopsyTreeChildren; private static final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source private static final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; @@ -476,8 +476,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } // if there's at least one image, load the image and open the top componen - autopsyTreeChildrenFactory = new AutopsyTreeChildrenFactory(); - autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); + autopsyTreeChildFactory = new AutopsyTreeChildFactory(); + autopsyTreeChildren = Children.create(autopsyTreeChildFactory, true); Node root = new AbstractNode(autopsyTreeChildren) { //JIRA-2807: What is the point of these overrides? /** @@ -909,7 +909,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } // refresh all children of the root. - autopsyTreeChildrenFactory.refreshChildren(); + autopsyTreeChildFactory.refreshChildren(); // Select the first node and reset the selection history // This should happen on the EDT once the tree has been rebuilt. From b205a983d3decf990ac1c21a27822abbfaa04b90 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 30 Jul 2018 11:24:54 -0400 Subject: [PATCH 94/96] Module version updates for release 4.8.0 --- Core/manifest.mf | 2 +- Core/nbproject/project.properties | 2 +- Core/nbproject/project.xml | 4 ++-- CoreLibs/manifest.mf | 4 ++-- Experimental/nbproject/project.xml | 4 ++-- ImageGallery/nbproject/project.xml | 4 ++-- KeywordSearch/manifest.mf | 2 +- KeywordSearch/nbproject/project.xml | 4 ++-- RecentActivity/manifest.mf | 2 +- RecentActivity/nbproject/project.xml | 2 +- thunderbirdparser/nbproject/project.xml | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Core/manifest.mf b/Core/manifest.mf index 67d3366e22..5eb077ef30 100644 --- a/Core/manifest.mf +++ b/Core/manifest.mf @@ -2,7 +2,7 @@ Manifest-Version: 1.0 OpenIDE-Module: org.sleuthkit.autopsy.core/10 OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/core/Bundle.properties OpenIDE-Module-Layer: org/sleuthkit/autopsy/core/layer.xml -OpenIDE-Module-Implementation-Version: 23 +OpenIDE-Module-Implementation-Version: 24 OpenIDE-Module-Requires: org.openide.windows.WindowManager AutoUpdate-Show-In-Client: true AutoUpdate-Essential-Module: true diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 9ff74aca14..80bb473653 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -46,5 +46,5 @@ nbm.homepage=http://www.sleuthkit.org/ nbm.module.author=Brian Carrier nbm.needs.restart=true source.reference.curator-recipes-2.8.0.jar=release/modules/ext/curator-recipes-2.8.0-sources.jar -spec.version.base=10.11 +spec.version.base=10.12 diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 46ad7c5426..859db9f0d2 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -251,7 +251,7 @@ 3 - 1.1 + 1.2 @@ -499,7 +499,7 @@ ext/xmpcore-5.1.3.jar release/modules/ext/xmpcore-5.1.3.jar - + ext/SparseBitSet-1.1.jar release/modules/ext/SparseBitSet-1.1.jar diff --git a/CoreLibs/manifest.mf b/CoreLibs/manifest.mf index e2386f67ce..24af339d11 100644 --- a/CoreLibs/manifest.mf +++ b/CoreLibs/manifest.mf @@ -1,8 +1,8 @@ Manifest-Version: 1.0 OpenIDE-Module: org.sleuthkit.autopsy.corelibs/3 -OpenIDE-Module-Implementation-Version: 4 +OpenIDE-Module-Implementation-Version: 5 OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/corelibs/Bundle.properties -OpenIDE-Module-Specification-Version: 1.1 +OpenIDE-Module-Specification-Version: 1.2 AutoUpdate-Show-In-Client: true AutoUpdate-Essential-Module: true diff --git a/Experimental/nbproject/project.xml b/Experimental/nbproject/project.xml index 6d3d3f6730..c0a18a9922 100644 --- a/Experimental/nbproject/project.xml +++ b/Experimental/nbproject/project.xml @@ -135,7 +135,7 @@ 10 - 10.11 + 10.12 @@ -144,7 +144,7 @@ 3 - 1.1 + 1.2 diff --git a/ImageGallery/nbproject/project.xml b/ImageGallery/nbproject/project.xml index 0fdad950f2..dcaa641e75 100644 --- a/ImageGallery/nbproject/project.xml +++ b/ImageGallery/nbproject/project.xml @@ -127,7 +127,7 @@ 10 - 10.11 + 10.12 @@ -136,7 +136,7 @@ 3 - 1.1 + 1.2 diff --git a/KeywordSearch/manifest.mf b/KeywordSearch/manifest.mf index 9f3126687e..5b53ffe61d 100644 --- a/KeywordSearch/manifest.mf +++ b/KeywordSearch/manifest.mf @@ -1,7 +1,7 @@ Manifest-Version: 1.0 AutoUpdate-Show-In-Client: true OpenIDE-Module: org.sleuthkit.autopsy.keywordsearch/6 -OpenIDE-Module-Implementation-Version: 19 +OpenIDE-Module-Implementation-Version: 20 OpenIDE-Module-Install: org/sleuthkit/autopsy/keywordsearch/Installer.class OpenIDE-Module-Layer: org/sleuthkit/autopsy/keywordsearch/layer.xml OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/keywordsearch/Bundle.properties diff --git a/KeywordSearch/nbproject/project.xml b/KeywordSearch/nbproject/project.xml index 25142d8119..343dc691e7 100644 --- a/KeywordSearch/nbproject/project.xml +++ b/KeywordSearch/nbproject/project.xml @@ -119,7 +119,7 @@ 10 - 10.11 + 10.12 @@ -128,7 +128,7 @@ 3 - 1.1 + 1.2 diff --git a/RecentActivity/manifest.mf b/RecentActivity/manifest.mf index 3121c6b8ca..c115996776 100644 --- a/RecentActivity/manifest.mf +++ b/RecentActivity/manifest.mf @@ -1,6 +1,6 @@ Manifest-Version: 1.0 OpenIDE-Module: org.sleuthkit.autopsy.recentactivity/6 -OpenIDE-Module-Implementation-Version: 15 +OpenIDE-Module-Implementation-Version: 16 OpenIDE-Module-Layer: org/sleuthkit/autopsy/recentactivity/layer.xml OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/recentactivity/Bundle.properties OpenIDE-Module-Requires: diff --git a/RecentActivity/nbproject/project.xml b/RecentActivity/nbproject/project.xml index fd72fa7dfe..4d85f94e9b 100644 --- a/RecentActivity/nbproject/project.xml +++ b/RecentActivity/nbproject/project.xml @@ -60,7 +60,7 @@ 10 - 10.11 + 10.12 diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml index 110c3b8ede..d4c0a0b53d 100644 --- a/thunderbirdparser/nbproject/project.xml +++ b/thunderbirdparser/nbproject/project.xml @@ -36,7 +36,7 @@ 10 - 10.11 + 10.12 From 830f33861136a4fa2253eaaa842861ebabd2fcb7 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 31 Jul 2018 10:23:06 -0400 Subject: [PATCH 95/96] Fixes for postgres queries --- .../autopsy/report/TableReportGenerator.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index f87851fe87..64f1f509af 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -582,7 +582,7 @@ class TableReportGenerator { adHocCountQuery += " AND (art.artifact_id = tag.artifact_id) AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS } adHocCountQuery += "EXCEPT " + // NON-NLS - "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ")) "; //NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ")) AS adHocHits"; //NON-NLS int adHocCount = 0; try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(adHocCountQuery)) { @@ -600,7 +600,7 @@ class TableReportGenerator { // Create the query to get the keyword list names if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { - orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS + orderByClause = "ORDER BY convert_to(list, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { orderByClause = "ORDER BY list ASC"; //NON-NLS } @@ -621,8 +621,9 @@ class TableReportGenerator { "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS } if (adHocCount > 0) { - keywordListQuery += " UNION SELECT \"\" AS list "; + keywordListQuery += " UNION SELECT \'\' AS list "; } + keywordListQuery = "SELECT * FROM ( " + keywordListQuery + " ) kwListNames "; keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS // Make the table of contents links for each list type @@ -645,17 +646,17 @@ class TableReportGenerator { BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getDisplayName())); } catch (TskCoreException | SQLException ex) { errorList.add(NbBundle.getMessage(this.getClass(), "ReportGenerator.errList.failedQueryKWLists")); - logger.log(Level.SEVERE, "Failed to query keyword lists: ", ex); //NON-NLS + logger.log(Level.SEVERE, "Failed to query keyword lists with query " + keywordListQuery, ex); //NON-NLS return; } // Query for keywords, grouped by list if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { - orderByClause = "ORDER BY convert_to(att3.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS - + "convert_to(att1.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS - + "convert_to(f.parent_path, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS - + "convert_to(f.name, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS - + "convert_to(att2.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS + orderByClause = "ORDER BY convert_to(list, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + + "convert_to(keyword, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + + "convert_to(parent_path, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + + "convert_to(name, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + + "convert_to(preview, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } @@ -684,7 +685,7 @@ class TableReportGenerator { // Query for keywords that are not part of a list String keywordAdHocQuery = - "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS + "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \'\' AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS "WHERE " + // NON-NLS " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " + // NON-NLS @@ -697,7 +698,7 @@ class TableReportGenerator { "AND (att2.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID() + ") " + // NON-NLS "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS - String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; + String keywordsQuery = "SELECT * FROM ( " + keywordListsQuery + " UNION " + keywordAdHocQuery + " ) kwHits " + orderByClause; try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); @@ -770,7 +771,7 @@ class TableReportGenerator { tableModule.endDataType(); } catch (TskCoreException | SQLException ex) { errorList.add(NbBundle.getMessage(this.getClass(), "ReportGenerator.errList.failedQueryKWs")); - logger.log(Level.SEVERE, "Failed to query keywords: ", ex); //NON-NLS + logger.log(Level.SEVERE, "Failed to query keywords with query " + keywordsQuery, ex); //NON-NLS } } From 4a637581a04d8b815ab39371a7fa96d46efd132c Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 1 Aug 2018 16:21:56 -0400 Subject: [PATCH 96/96] Change build type back to DEVELOPMENT --- nbproject/project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbproject/project.properties b/nbproject/project.properties index 1ce749b501..55a5b67cfd 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -6,8 +6,8 @@ app.name=${branding.token} ### if left unset, version will default to today's date app.version=4.8.0 ### build.type must be one of: DEVELOPMENT, RELEASE -build.type=RELEASE -#build.type=DEVELOPMENT +#build.type=RELEASE +build.type=DEVELOPMENT project.org.netbeans.progress=org-netbeans-api-progress project.org.sleuthkit.autopsy.experimental=Experimental