codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-10-05 08:46:20 +00:00
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into 2823_tskLayerTest

Browse Source
This commit is contained in:
Ann Priestman
2018-06-08 11:30:43 -04:00
parent c0ea7776e5 a742fcbf5d
commit a2c998479e
77 changed files with 3986 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core/build.xml
+8 -4
View File
@@ -89,22 +89,26 @@
<target name="getTestDataFiles">
<mkdir dir="${basedir}/test/qa-functional/data"/>
<get src="https://drive.google.com/uc?id=1dLYGctuvRQMmnzfXPppTM_9gB49eLc_g" dest="${test-input}/embedded.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1dLYGctuvRQMmnzfXPppTM_9gB49eLc_g" dest="${test-input}/embedded.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1JACMDyH4y54ypGzFWl82ZzMQf3qbrioP" dest="${test-input}/encryption_detection_bitlocker_test.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=17sGybvmBGsWWJYo1IWKmO04oG9hKpPi3" dest="${test-input}/encryption_detection_sqlcipher_test.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=0BxdBkzm5VKGNT0dGY0dqcHVsU3M" dest="${test-input}/filter_test1.img" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1bghoSm7z7nhmGIxlllyY1MMlbLntxm7n" dest="${test-input}/local_files_test.zip" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1BrSiUQ1fzxFS9vIaK4mYKX6qIVp9kRWT" dest="${test-input}/password_detection_test.img" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1HD8s4rculgHV1qZT5g80Kg7j4m1qccrN" dest="${test-input}/veracrypt_detection_test.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1O5D09fFCFpXZqw0uLEs8kVLtfYTxqXAd" dest="${test-input}/commonfiles_image1_v1.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1rMP1QTI0LdppzdypbG-4BDwkKcR3tHXc" dest="${test-input}/commonfiles_image2_v1.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1OdwyJ2lru55ZPdvwzj3pq6sXIys27i4x" dest="${test-input}/commonfiles_image3_v1.vhd" skipexisting="true"/>
<get src="https://drive.google.com/uc?id=1GoF2x0km5AyFvE926ttN20lrMX1oLN7E" dest="${test-input}/commonfiles_image4_v1.vhd" skipexisting="true"/>
</target>
<target name="get-deps" depends="init-ivy,getTSKJars,get-thirdparty-dependencies,get-InternalPythonModules, download-binlist,getTestDataFiles">
<mkdir dir="${ext.dir}"/>
<copy file="${thirdparty.dir}/LICENSE-2.0.txt" todir="${ext.dir}" />
<copy file="${thirdparty.dir}/LICENSE-2.0.txt" todir="${ext.dir}" />
<!-- fetch all the dependencies from Ivy and stick them in the right places -->
<ivy:resolve log="quiet"/>
<ivy:retrieve conf="core" pattern="${ext.dir}/[artifact]-[revision](-[classifier]).[ext]" />
</target>
</target>
<target name="init" depends="get-deps,harness.init"/>
Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/ArtifactKey.java
-70
View File
@@ -1,70 +0,0 @@
/*
* Central Repository
*
* Copyright 2015-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.centralrepository.contentviewer;
import java.util.Objects;
/**
* Used as a key to ensure we eliminate duplicates from the result set by not overwriting CR correlation instances.
*/
final class ArtifactKey {
private final String dataSourceID;
private final String filePath;
ArtifactKey(String theDataSource, String theFilePath) {
dataSourceID = theDataSource;
filePath = theFilePath.toLowerCase();
}
/**
*
* @return the dataSourceID device ID
*/
String getDataSourceID() {
return dataSourceID;
}
/**
*
* @return the filPath including the filename and extension.
*/
String getFilePath() {
return filePath;
}
@Override
public boolean equals(Object other) {
if (other instanceof ArtifactKey) {
return ((ArtifactKey) other).getDataSourceID().equals(dataSourceID) && ((ArtifactKey) other).getFilePath().equals(filePath);
}
return false;
}
@Override
public int hashCode() {
//int hash = 7;
//hash = 67 * hash + this.dataSourceID.hashCode();
//hash = 67 * hash + this.filePath.hashCode();
return Objects.hash(dataSourceID, filePath);
}
}
Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java
+187 -97
View File
@@ -18,7 +18,9 @@
*/
package org.sleuthkit.autopsy.centralrepository.contentviewer;
import java.awt.Color;
import java.awt.Component;
import java.awt.Dimension;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.BufferedWriter;
@@ -31,19 +33,30 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import java.util.stream.Collectors;
import javax.swing.GroupLayout;
import javax.swing.JFileChooser;
import javax.swing.JLabel;
import javax.swing.JMenuItem;
import javax.swing.JOptionPane;
import static javax.swing.JOptionPane.DEFAULT_OPTION;
import static javax.swing.JOptionPane.PLAIN_MESSAGE;
import static javax.swing.JOptionPane.ERROR_MESSAGE;
import javax.swing.JPanel;
import javax.swing.JPopupMenu;
import javax.swing.JScrollPane;
import javax.swing.JTable;
import javax.swing.LayoutStyle;
import javax.swing.ListSelectionModel;
import javax.swing.filechooser.FileNameExtensionFilter;
import javax.swing.table.TableCellRenderer;
import javax.swing.table.TableColumn;
import org.openide.awt.Mnemonics;
import org.openide.nodes.Node;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.openide.util.lookup.ServiceProvider;
import org.sleuthkit.autopsy.casemodule.Case;
@@ -65,15 +78,17 @@ import org.sleuthkit.datamodel.TskException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskDataException;
/**
* View correlation results from other cases
*/
@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives
@ServiceProvider(service = DataContentViewer.class, position = 8)
@Messages({"DataContentViewerOtherCases.title=Other Occurrences",
"DataContentViewerOtherCases.toolTip=Displays instances of the selected file/artifact from other occurrences.",})
public class DataContentViewerOtherCases extends javax.swing.JPanel implements DataContentViewer {
public class DataContentViewerOtherCases extends JPanel implements DataContentViewer {
private final static Logger LOGGER = Logger.getLogger(DataContentViewerOtherCases.class.getName());
private final DataContentViewerOtherCasesTableModel tableModel;
@@ -401,7 +416,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
// correlate on blackboard artifact attributes if they exist and supported
BlackboardArtifact bbArtifact = getBlackboardArtifactFromNode(node);
if (bbArtifact != null) {
if (bbArtifact != null && EamDb.isEnabled()) {
ret.addAll(EamArtifactUtil.getCorrelationAttributeFromBlackboardArtifact(bbArtifact, false, false));
}
@@ -451,12 +466,12 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
*
* @return A collection of correlated artifact instances from other cases
*/
private Map<ArtifactKey, CorrelationAttributeInstance> getCorrelatedInstances(CorrelationAttribute corAttr, String dataSourceName, String deviceId) {
private Map<UniquePathKey,CorrelationAttributeInstance> getCorrelatedInstances(CorrelationAttribute corAttr, String dataSourceName, String deviceId) {
// @@@ Check exception
try {
final Case openCase = Case.getCurrentCase();
String caseUUID = openCase.getName();
HashMap<ArtifactKey, CorrelationAttributeInstance> artifactInstances = new HashMap<>();
HashMap<UniquePathKey,CorrelationAttributeInstance> artifactInstances = new HashMap<>();
if (EamDb.isEnabled()) {
EamDb dbManager = EamDb.getInstance();
@@ -464,8 +479,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
.filter(artifactInstance -> !artifactInstance.getCorrelationCase().getCaseUUID().equals(caseUUID)
|| !artifactInstance.getCorrelationDataSource().getName().equals(dataSourceName)
|| !artifactInstance.getCorrelationDataSource().getDeviceID().equals(deviceId))
.collect(Collectors.toMap(
correlationAttr -> new ArtifactKey(correlationAttr.getCorrelationDataSource().getDeviceID(), correlationAttr.getFilePath()),
.collect(Collectors.toMap(correlationAttr -> new UniquePathKey(correlationAttr.getCorrelationDataSource().getDeviceID(), correlationAttr.getFilePath()),
correlationAttr -> correlationAttr)));
}
@@ -507,29 +521,59 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
}
private void addOrUpdateAttributeInstance(final Case openCase, Map<ArtifactKey, CorrelationAttributeInstance> artifactInstances, AbstractFile caseDbFile) throws TskCoreException, EamDbException {
CorrelationCase caze = new CorrelationCase(openCase.getNumber(), openCase.getDisplayName());
CorrelationDataSource dataSource = CorrelationDataSource.fromTSKDataSource(caze, caseDbFile.getDataSource());
String filePath = caseDbFile.getParentPath() + caseDbFile.getName();
ArtifactKey instKey = new ArtifactKey(dataSource.getDeviceID(), filePath);
CorrelationAttributeInstance caseDbInstance = new CorrelationAttributeInstance(caze, dataSource, filePath, "", caseDbFile.getKnown());
TskData.FileKnown knownStatus = caseDbInstance.getKnownStatus();
// If not known, check Tags for known and set
TskData.FileKnown knownBad = TskData.FileKnown.BAD;
if (!knownStatus.equals(knownBad)) {
List<ContentTag> fileMatchTags = openCase.getServices().getTagsManager().getContentTagsByContent(caseDbFile);
/**
* Adds the file to the artifactInstances map if it does not already exist
*
* @param autopsyCase
* @param artifactInstances
* @param newFile
* @throws TskCoreException
* @throws EamDbException
*/
private void addOrUpdateAttributeInstance(final Case autopsyCase, Map<UniquePathKey,CorrelationAttributeInstance> artifactInstances, AbstractFile newFile) throws TskCoreException, EamDbException {
// figure out if the casedb file is known via either hash or tags
TskData.FileKnown localKnown = newFile.getKnown();
if (localKnown != TskData.FileKnown.BAD) {
List<ContentTag> fileMatchTags = autopsyCase.getServices().getTagsManager().getContentTagsByContent(newFile);
for (ContentTag tag : fileMatchTags) {
TskData.FileKnown tagKnownStatus = tag.getName().getKnownStatus();
if (tagKnownStatus.equals(knownBad)) {
caseDbInstance.setKnownStatus(knownBad);
if (tagKnownStatus.equals(TskData.FileKnown.BAD)) {
localKnown = TskData.FileKnown.BAD;
break;
}
}
}
// If known, or not in CR, add
if (caseDbInstance.getKnownStatus().equals(knownBad) || !artifactInstances.containsKey(instKey)) {
artifactInstances.put(instKey, caseDbInstance);
// make a key to see if the file is already in the map
String filePath = newFile.getParentPath() + newFile.getName();
String deviceId;
try {
deviceId = autopsyCase.getSleuthkitCase().getDataSource(newFile.getDataSource().getId()).getDeviceId();
} catch (TskDataException | TskCoreException ex) {
LOGGER.log(Level.WARNING, "Error getting data source info: " + ex);
return;
}
UniquePathKey uniquePathKey = new UniquePathKey(deviceId, filePath);
// double check that the CR version is BAD if the caseDB version is BAD.
if (artifactInstances.containsKey(uniquePathKey)) {
if (localKnown == TskData.FileKnown.BAD) {
CorrelationAttributeInstance prevInstance = artifactInstances.get(uniquePathKey);
prevInstance.setKnownStatus(localKnown);
}
}
// add the data from the case DB by pushing data into CorrelationAttributeInstance class
else {
// NOTE: If we are in here, it is likely because CR is not enabled. So, we cannot rely
// on any of the methods that query the DB.
CorrelationCase correlationCase = new CorrelationCase(autopsyCase.getName(), autopsyCase.getDisplayName());
CorrelationDataSource correlationDataSource = CorrelationDataSource.fromTSKDataSource(correlationCase, newFile.getDataSource());
CorrelationAttributeInstance caseDbInstance = new CorrelationAttributeInstance(correlationCase, correlationDataSource, filePath, "", localKnown);
artifactInstances.put(uniquePathKey, caseDbInstance);
}
}
@@ -539,9 +583,15 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
// Is supported if this node
// has correlatable content (File, BlackboardArtifact) OR
// other common files across datasources.
return this.file != null
if(EamDb.isEnabled()){
return this.file != null
&& this.file.getSize() > 0
&& !getCorrelationAttributesFromNode(node).isEmpty();
} else{
return this.file != null
&& this.file.getSize() > 0;
}
}
@Override
@@ -582,7 +632,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
// get the attributes we can correlate on
correlationAttributes.addAll(getCorrelationAttributesFromNode(node));
for (CorrelationAttribute corAttr : correlationAttributes) {
Map<ArtifactKey, CorrelationAttributeInstance> corAttrInstances = new HashMap<>(0);
Map<UniquePathKey, CorrelationAttributeInstance> corAttrInstances = new HashMap<>(0);
// get correlation and reference set instances from DB
corAttrInstances.putAll(getCorrelatedInstances(corAttr, dataSourceName, deviceId));
@@ -602,6 +652,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
}
if (correlationAttributes.isEmpty()) {
// @@@ BC: We should have a more descriptive message than this. Mention that the file didn't have a MD5, etc.
displayMessageOnTableStatusPanel(Bundle.DataContentViewerOtherCases_table_noArtifacts());
} else if (0 == tableModel.getRowCount()) {
displayMessageOnTableStatusPanel(Bundle.DataContentViewerOtherCases_table_isempty());
@@ -639,131 +690,170 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
rightClickPopupMenu = new javax.swing.JPopupMenu();
selectAllMenuItem = new javax.swing.JMenuItem();
exportToCSVMenuItem = new javax.swing.JMenuItem();
showCaseDetailsMenuItem = new javax.swing.JMenuItem();
showCommonalityMenuItem = new javax.swing.JMenuItem();
CSVFileChooser = new javax.swing.JFileChooser();
otherCasesPanel = new javax.swing.JPanel();
tableContainerPanel = new javax.swing.JPanel();
tableScrollPane = new javax.swing.JScrollPane();
otherCasesTable = new javax.swing.JTable();
tableStatusPanel = new javax.swing.JPanel();
tableStatusPanelLabel = new javax.swing.JLabel();
rightClickPopupMenu = new JPopupMenu();
selectAllMenuItem = new JMenuItem();
exportToCSVMenuItem = new JMenuItem();
showCaseDetailsMenuItem = new JMenuItem();
showCommonalityMenuItem = new JMenuItem();
CSVFileChooser = new JFileChooser();
otherCasesPanel = new JPanel();
tableContainerPanel = new JPanel();
tableScrollPane = new JScrollPane();
otherCasesTable = new JTable();
tableStatusPanel = new JPanel();
tableStatusPanelLabel = new JLabel();
org.openide.awt.Mnemonics.setLocalizedText(selectAllMenuItem, org.openide.util.NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.selectAllMenuItem.text")); // NOI18N
Mnemonics.setLocalizedText(selectAllMenuItem, NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.selectAllMenuItem.text")); // NOI18N
rightClickPopupMenu.add(selectAllMenuItem);
org.openide.awt.Mnemonics.setLocalizedText(exportToCSVMenuItem, org.openide.util.NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.exportToCSVMenuItem.text")); // NOI18N
Mnemonics.setLocalizedText(exportToCSVMenuItem, NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.exportToCSVMenuItem.text")); // NOI18N
rightClickPopupMenu.add(exportToCSVMenuItem);
org.openide.awt.Mnemonics.setLocalizedText(showCaseDetailsMenuItem, org.openide.util.NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.showCaseDetailsMenuItem.text")); // NOI18N
Mnemonics.setLocalizedText(showCaseDetailsMenuItem, NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.showCaseDetailsMenuItem.text")); // NOI18N
rightClickPopupMenu.add(showCaseDetailsMenuItem);
org.openide.awt.Mnemonics.setLocalizedText(showCommonalityMenuItem, org.openide.util.NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.showCommonalityMenuItem.text")); // NOI18N
Mnemonics.setLocalizedText(showCommonalityMenuItem, NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.showCommonalityMenuItem.text")); // NOI18N
rightClickPopupMenu.add(showCommonalityMenuItem);
setMinimumSize(new java.awt.Dimension(1500, 10));
setMinimumSize(new Dimension(1500, 10));
setOpaque(false);
setPreferredSize(new java.awt.Dimension(1500, 44));
setPreferredSize(new Dimension(1500, 44));
otherCasesPanel.setPreferredSize(new java.awt.Dimension(1500, 144));
otherCasesPanel.setPreferredSize(new Dimension(1500, 144));
tableContainerPanel.setPreferredSize(new java.awt.Dimension(1500, 63));
tableContainerPanel.setPreferredSize(new Dimension(1500, 63));
tableScrollPane.setPreferredSize(new java.awt.Dimension(1500, 30));
tableScrollPane.setPreferredSize(new Dimension(1500, 30));
otherCasesTable.setAutoCreateRowSorter(true);
otherCasesTable.setModel(tableModel);
otherCasesTable.setToolTipText(org.openide.util.NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.table.toolTip.text")); // NOI18N
otherCasesTable.setToolTipText(NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.table.toolTip.text")); // NOI18N
otherCasesTable.setComponentPopupMenu(rightClickPopupMenu);
otherCasesTable.setSelectionMode(javax.swing.ListSelectionModel.SINGLE_INTERVAL_SELECTION);
otherCasesTable.setSelectionMode(ListSelectionModel.SINGLE_INTERVAL_SELECTION);
tableScrollPane.setViewportView(otherCasesTable);
tableStatusPanel.setPreferredSize(new java.awt.Dimension(1500, 16));
tableStatusPanel.setPreferredSize(new Dimension(1500, 16));
tableStatusPanelLabel.setForeground(new java.awt.Color(255, 0, 51));
tableStatusPanelLabel.setForeground(new Color(255, 0, 51));
javax.swing.GroupLayout tableStatusPanelLayout = new javax.swing.GroupLayout(tableStatusPanel);
GroupLayout tableStatusPanelLayout = new GroupLayout(tableStatusPanel);
tableStatusPanel.setLayout(tableStatusPanelLayout);
tableStatusPanelLayout.setHorizontalGroup(
tableStatusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
tableStatusPanelLayout.setHorizontalGroup(tableStatusPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGap(0, 0, Short.MAX_VALUE)
.addGroup(tableStatusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(tableStatusPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGroup(tableStatusPanelLayout.createSequentialGroup()
.addContainerGap()
.addComponent(tableStatusPanelLabel, javax.swing.GroupLayout.DEFAULT_SIZE, 780, Short.MAX_VALUE)
.addComponent(tableStatusPanelLabel, GroupLayout.DEFAULT_SIZE, 780, Short.MAX_VALUE)
.addContainerGap()))
);
tableStatusPanelLayout.setVerticalGroup(
tableStatusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
tableStatusPanelLayout.setVerticalGroup(tableStatusPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGap(0, 16, Short.MAX_VALUE)
.addGroup(tableStatusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(tableStatusPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGroup(tableStatusPanelLayout.createSequentialGroup()
.addComponent(tableStatusPanelLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 16, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(tableStatusPanelLabel, GroupLayout.PREFERRED_SIZE, 16, GroupLayout.PREFERRED_SIZE)
.addGap(0, 0, Short.MAX_VALUE)))
);
javax.swing.GroupLayout tableContainerPanelLayout = new javax.swing.GroupLayout(tableContainerPanel);
GroupLayout tableContainerPanelLayout = new GroupLayout(tableContainerPanel);
tableContainerPanel.setLayout(tableContainerPanelLayout);
tableContainerPanelLayout.setHorizontalGroup(
tableContainerPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(tableScrollPane, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(tableStatusPanel, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
tableContainerPanelLayout.setHorizontalGroup(tableContainerPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addComponent(tableScrollPane, GroupLayout.Alignment.TRAILING, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(tableStatusPanel, GroupLayout.Alignment.TRAILING, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
);
tableContainerPanelLayout.setVerticalGroup(
tableContainerPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
tableContainerPanelLayout.setVerticalGroup(tableContainerPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGroup(tableContainerPanelLayout.createSequentialGroup()
.addComponent(tableScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(tableStatusPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(tableScrollPane, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addPreferredGap(LayoutStyle.ComponentPlacement.RELATED)
.addComponent(tableStatusPanel, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE)
.addContainerGap())
);
javax.swing.GroupLayout otherCasesPanelLayout = new javax.swing.GroupLayout(otherCasesPanel);
GroupLayout otherCasesPanelLayout = new GroupLayout(otherCasesPanel);
otherCasesPanel.setLayout(otherCasesPanelLayout);
otherCasesPanelLayout.setHorizontalGroup(
otherCasesPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
otherCasesPanelLayout.setHorizontalGroup(otherCasesPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGap(0, 1500, Short.MAX_VALUE)
.addGroup(otherCasesPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(tableContainerPanel, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGroup(otherCasesPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addComponent(tableContainerPanel, GroupLayout.Alignment.TRAILING, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
);
otherCasesPanelLayout.setVerticalGroup(
otherCasesPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
otherCasesPanelLayout.setVerticalGroup(otherCasesPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGap(0, 60, Short.MAX_VALUE)
.addGroup(otherCasesPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(otherCasesPanelLayout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addGroup(otherCasesPanelLayout.createSequentialGroup()
.addComponent(tableContainerPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 60, Short.MAX_VALUE)
.addComponent(tableContainerPanel, GroupLayout.DEFAULT_SIZE, 60, Short.MAX_VALUE)
.addGap(0, 0, 0)))
);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
GroupLayout layout = new GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(otherCasesPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
layout.setHorizontalGroup(layout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addComponent(otherCasesPanel, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(otherCasesPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 60, Short.MAX_VALUE)
layout.setVerticalGroup(layout.createParallelGroup(GroupLayout.Alignment.LEADING)
.addComponent(otherCasesPanel, GroupLayout.DEFAULT_SIZE, 60, Short.MAX_VALUE)
);
}// </editor-fold>//GEN-END:initComponents
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JFileChooser CSVFileChooser;
private javax.swing.JMenuItem exportToCSVMenuItem;
private javax.swing.JPanel otherCasesPanel;
private javax.swing.JTable otherCasesTable;
private javax.swing.JPopupMenu rightClickPopupMenu;
private javax.swing.JMenuItem selectAllMenuItem;
private javax.swing.JMenuItem showCaseDetailsMenuItem;
private javax.swing.JMenuItem showCommonalityMenuItem;
private javax.swing.JPanel tableContainerPanel;
private javax.swing.JScrollPane tableScrollPane;
private javax.swing.JPanel tableStatusPanel;
private javax.swing.JLabel tableStatusPanelLabel;
private JFileChooser CSVFileChooser;
private JMenuItem exportToCSVMenuItem;
private JPanel otherCasesPanel;
private JTable otherCasesTable;
private JPopupMenu rightClickPopupMenu;
private JMenuItem selectAllMenuItem;
private JMenuItem showCaseDetailsMenuItem;
private JMenuItem showCommonalityMenuItem;
private JPanel tableContainerPanel;
private JScrollPane tableScrollPane;
private JPanel tableStatusPanel;
private JLabel tableStatusPanelLabel;
// End of variables declaration//GEN-END:variables
/**
* Used as a key to ensure we eliminate duplicates from the result set by not overwriting CR correlation instances.
*/
static final class UniquePathKey {
private final String dataSourceID;
private final String filePath;
UniquePathKey(String theDataSource, String theFilePath) {
super();
dataSourceID = theDataSource;
filePath = theFilePath.toLowerCase();
}
/**
*
* @return the dataSourceID device ID
*/
String getDataSourceID() {
return dataSourceID;
}
/**
*
* @return the filPath including the filename and extension.
*/
String getFilePath() {
return filePath;
}
@Override
public boolean equals(Object other) {
if (other instanceof UniquePathKey) {
return ((UniquePathKey) other).getDataSourceID().equals(dataSourceID) && ((UniquePathKey) other).getFilePath().equals(filePath);
}
return false;
}
@Override
public int hashCode() {
//int hash = 7;
//hash = 67 * hash + this.dataSourceID.hashCode();
//hash = 67 * hash + this.filePath.hashCode();
return Objects.hash(dataSourceID, filePath);
}
}
}
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java
+6
View File
@@ -39,6 +39,8 @@ import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import static org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil.updateSchemaVersion;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.healthmonitor.EnterpriseHealthMonitor;
import org.sleuthkit.autopsy.healthmonitor.TimingMetric;
import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
import org.sleuthkit.datamodel.TskData;
@@ -938,6 +940,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
if (bulkArtifactsCount == 0) {
return;
}
TimingMetric timingMetric = EnterpriseHealthMonitor.getTimingMetric("Correlation Engine: Bulk insert");
for (CorrelationAttribute.Type type : artifactTypes) {
@@ -988,6 +992,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
bulkPs.executeBatch();
bulkArtifacts.get(type.getDbTableName()).clear();
}
EnterpriseHealthMonitor.submitTimingMetric(timingMetric);
// Reset state
bulkArtifactsCount = 0;
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java
+2 -8
View File
@@ -226,13 +226,10 @@ public class EamArtifactUtil {
* Does not add the artifact to the database.
*
* @param content The content object
* @param knownStatus Unknown, notable, or known
* @param comment The comment for the new artifact (generally used for a
* tag comment)
*
* @return The new EamArtifact or null if creation failed
*/
public static CorrelationAttribute getCorrelationAttributeFromContent(Content content, TskData.FileKnown knownStatus, String comment) {
public static CorrelationAttribute makeCorrelationAttributeFromContent(Content content) {
if (!(content instanceof AbstractFile)) {
return null;
@@ -261,10 +258,7 @@ public class EamArtifactUtil {
CorrelationAttributeInstance cei = new CorrelationAttributeInstance(
correlationCase,
CorrelationDataSource.fromTSKDataSource(correlationCase, af.getDataSource()),
af.getParentPath() + af.getName(),
comment,
knownStatus
);
af.getParentPath() + af.getName());
eamArtifact.addInstance(cei);
return eamArtifact;
} catch (TskCoreException | EamDbException | NoCurrentCaseException ex) {
Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java
+2 -4
View File
@@ -192,8 +192,7 @@ final class CaseEventListener implements PropertyChangeListener {
}
}
final CorrelationAttribute eamArtifact = EamArtifactUtil.getCorrelationAttributeFromContent(af,
knownStatus, comment);
final CorrelationAttribute eamArtifact = EamArtifactUtil.makeCorrelationAttributeFromContent(af);
if (eamArtifact != null) {
// send update to Central Repository db
@@ -402,8 +401,7 @@ final class CaseEventListener implements PropertyChangeListener {
}
//if the file will have no tags with a status which would prevent the current status from being changed
if (!hasTagWithConflictingKnownStatus) {
final CorrelationAttribute eamArtifact = EamArtifactUtil.getCorrelationAttributeFromContent(contentTag.getContent(),
tagName.getKnownStatus(), "");
final CorrelationAttribute eamArtifact = EamArtifactUtil.makeCorrelationAttributeFromContent(contentTag.getContent());
if (eamArtifact != null) {
EamDb.getInstance().setArtifactInstanceKnownStatus(eamArtifact, tagName.getKnownStatus());
}
Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/NewArtifactsRunner.java
-61
View File
@@ -1,61 +0,0 @@
/*
* Central Repository
*
* Copyright 2015-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.centralrepository.eventlisteners;
import java.util.ArrayList;
import java.util.Collection;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
/**
* Thread to insert a new artifact into remote DB.
*/
public class NewArtifactsRunner implements Runnable {
private static final Logger LOGGER = Logger.getLogger(NewArtifactsRunner.class.getName());
private static final long serialVersionUID = 1L;
private final Collection<CorrelationAttribute> eamArtifacts;
@SuppressWarnings(value = {"unchecked", "rawtypes"})
public NewArtifactsRunner(Collection<CorrelationAttribute> eamArtifacts) {
this.eamArtifacts = new ArrayList(eamArtifacts);
}
@Override
public void run() {
if (!EamDb.isEnabled()) {
LOGGER.log(Level.WARNING, "Central Repository database not configured"); // NON-NLS
return;
}
try {
EamDb dbManager = EamDb.getInstance();
for (CorrelationAttribute eamArtifact : eamArtifacts) {
dbManager.addArtifact(eamArtifact);
}
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS
}
}
}
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java
+4
View File
@@ -49,6 +49,8 @@ import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListener;
import org.sleuthkit.autopsy.healthmonitor.EnterpriseHealthMonitor;
import org.sleuthkit.autopsy.healthmonitor.TimingMetric;
/**
* Ingest module for inserting entries into the Central Repository database on
@@ -129,7 +131,9 @@ final class IngestModule implements FileIngestModule {
*/
if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
try {
TimingMetric timingMetric = EnterpriseHealthMonitor.getTimingMetric("Correlation Engine: Notable artifact query");
List<String> caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
EnterpriseHealthMonitor.submitTimingMetric(timingMetric);
if (!caseDisplayNamesList.isEmpty()) {
postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList);
}
Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSourcesCommonFilesAlgorithm.java
+2 -2
View File
@@ -24,7 +24,7 @@ import java.util.Map;
/**
* Provides logic for selecting common files from all data sources.
*/
final class AllDataSourcesCommonFilesAlgorithm extends CommonFilesMetadataBuilder {
final public class AllDataSourcesCommonFilesAlgorithm extends CommonFilesMetadataBuilder {
private static final String WHERE_CLAUSE = "%s md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL)%s GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; //NON-NLS
@@ -36,7 +36,7 @@ final class AllDataSourcesCommonFilesAlgorithm extends CommonFilesMetadataBuilde
* @param filterByMediaMimeType match only on files whose mime types can be broadly categorized as media types
* @param filterByDocMimeType match only on files whose mime types can be broadly categorized as document types
*/
AllDataSourcesCommonFilesAlgorithm(Map<Long, String> dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) {
public AllDataSourcesCommonFilesAlgorithm(Map<Long, String> dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) {
super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType);
}
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadata.java
+3 -3
View File
@@ -26,7 +26,7 @@ import java.util.Map;
* Utility and wrapper model around data required for Common Files Search results.
* Subclass this to implement different selections of files from the case.
*/
final class CommonFilesMetadata {
final public class CommonFilesMetadata {
private final Map<String, Md5Metadata> metadata;
@@ -52,7 +52,7 @@ final class CommonFilesMetadata {
return this.metadata.get(md5);
}
Map<String, Md5Metadata> getMetadata() {
public Map<String, Md5Metadata> getMetadata() {
return Collections.unmodifiableMap(this.metadata);
}
@@ -60,7 +60,7 @@ final class CommonFilesMetadata {
* How many distinct file instances exist for this metadata?
* @return number of file instances
*/
int size() {
public int size() {
int count = 0;
for (Md5Metadata data : this.metadata.values()) {
count += data.size();
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java
+1 -1
View File
@@ -46,7 +46,7 @@ import org.sleuthkit.datamodel.TskCoreException;
* This entire thing runs on a background thread where exceptions are handled.
*/
@SuppressWarnings("PMD.AbstractNaming")
abstract class CommonFilesMetadataBuilder {
public abstract class CommonFilesMetadataBuilder {
private final Map<Long, String> dataSourceIdToNameMap;
private final boolean filterByMedia;
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java
+6 -55
View File
@@ -18,12 +18,9 @@
*/
package org.sleuthkit.autopsy.commonfilesearch;
import java.io.File;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
@@ -33,7 +30,6 @@ import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.openide.explorer.ExplorerManager;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataResultViewer;
import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent;
@@ -42,9 +38,6 @@ import org.sleuthkit.autopsy.corecomponents.TableFilterNode;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.directorytree.DataResultFilterNode;
import org.sleuthkit.autopsy.directorytree.DirectoryTreeTopComponent;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.SleuthkitCase.CaseDbQuery;
import org.sleuthkit.datamodel.TskCoreException;
/**
@@ -77,7 +70,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
initComponents();
this.setupDataSources();
this.errorText.setVisible(false);
}
@@ -98,10 +91,6 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
new SwingWorker<Map<Long, String>, Void>() {
private static final String SELECT_DATA_SOURCES_LOGICAL = "select obj_id, name from tsk_files where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))";
private static final String SELECT_DATA_SOURCES_IMAGE = "select obj_id, name from tsk_image_names where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))";
private void updateUi() {
String[] dataSourcesNames = new String[CommonFilesPanel.this.dataSourceMap.size()];
@@ -132,48 +121,10 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
return CommonFilesPanel.this.dataSourceMap.size() >= 2;
}
private void loadLogicalSources(SleuthkitCase tskDb, Map<Long, String> dataSouceMap) throws TskCoreException, SQLException {
//try block releases resources - exceptions are handled in done()
try (
CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_LOGICAL);
ResultSet resultSet = query.getResultSet()) {
while (resultSet.next()) {
Long objectId = resultSet.getLong(1);
String dataSourceName = resultSet.getString(2);
dataSouceMap.put(objectId, dataSourceName);
}
}
}
private void loadImageSources(SleuthkitCase tskDb, Map<Long, String> dataSouceMap) throws SQLException, TskCoreException {
//try block releases resources - exceptions are handled in done()
try (
CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_IMAGE);
ResultSet resultSet = query.getResultSet()) {
while (resultSet.next()) {
Long objectId = resultSet.getLong(1);
String dataSourceName = resultSet.getString(2);
File image = new File(dataSourceName);
String dataSourceNameTrimmed = image.getName();
dataSouceMap.put(objectId, dataSourceNameTrimmed);
}
}
}
@Override
protected Map<Long, String> doInBackground() throws NoCurrentCaseException, TskCoreException, SQLException {
Map<Long, String> dataSouceMap = new HashMap<>();
Case currentCase = Case.getCurrentCaseThrows();
SleuthkitCase tskDb = currentCase.getSleuthkitCase();
loadLogicalSources(tskDb, dataSouceMap);
loadImageSources(tskDb, dataSouceMap);
return dataSouceMap;
DataSourceLoader loader = new DataSourceLoader();
return loader.getDataSourceMap();
}
@Override
@@ -296,10 +247,10 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
TableFilterNode tableFilterWithDescendantsNode = new TableFilterNode(dataResultFilterNode);
DataResultViewerTable table = new DataResultViewerTable();
Collection<DataResultViewer> viewers = new ArrayList<>(1);
viewers.add(table);
DataResultTopComponent.createInstance(tabTitle, pathText, tableFilterWithDescendantsNode, metadata.size(), viewers);
} catch (InterruptedException ex) {
@@ -591,7 +542,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
this.pictureVideoCheckbox.setEnabled(true);
this.documentsCheckbox.setEnabled(true);
this.toggleErrorTextAndSearchBox();
}
}
Core/src/org/sleuthkit/autopsy/commonfilesearch/DataSourceLoader.java
+95
View File
@@ -0,0 +1,95 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilesearch;
import java.io.File;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Encapsulates logic required to create a mapping of data sources in the
* current case to their data source IDs.
*
* Intended to be used within the context of a SwingWorker or other background
* thread.
*/
public class DataSourceLoader {
private static final String SELECT_DATA_SOURCES_LOGICAL = "select obj_id, name from tsk_files where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))";
private static final String SELECT_DATA_SOURCES_IMAGE = "select obj_id, name from tsk_image_names where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))";
private void loadLogicalSources(SleuthkitCase tskDb, Map<Long, String> dataSouceMap) throws TskCoreException, SQLException {
//try block releases resources - exceptions are handled in done()
try (
SleuthkitCase.CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_LOGICAL);
ResultSet resultSet = query.getResultSet()
) {
while (resultSet.next()) {
Long objectId = resultSet.getLong(1);
String dataSourceName = resultSet.getString(2);
dataSouceMap.put(objectId, dataSourceName);
}
}
}
private void loadImageSources(SleuthkitCase tskDb, Map<Long, String> dataSouceMap) throws SQLException, TskCoreException {
//try block releases resources - exceptions are handled in done()
try (
SleuthkitCase.CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_IMAGE);
ResultSet resultSet = query.getResultSet()) {
while (resultSet.next()) {
Long objectId = resultSet.getLong(1);
String dataSourceName = resultSet.getString(2);
File image = new File(dataSourceName);
String dataSourceNameTrimmed = image.getName();
dataSouceMap.put(objectId, dataSourceNameTrimmed);
}
}
}
/**
* Get a map of data source Ids to their string names for the current case.
*
* @return Map of Long (id) to String (name)
* @throws NoCurrentCaseException
* @throws TskCoreException
* @throws SQLException
*/
public Map<Long, String> getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException {
Map<Long, String> dataSouceMap = new HashMap<>();
Case currentCase = Case.getCurrentCaseThrows();
SleuthkitCase tskDb = currentCase.getSleuthkitCase();
loadLogicalSources(tskDb, dataSouceMap);
loadImageSources(tskDb, dataSouceMap);
return dataSouceMap;
}
}
Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java
+6 -4
View File
@@ -24,7 +24,7 @@ import java.util.Map;
/**
* Provides logic for selecting common files from a single data source.
*/
final class SingleDataSource extends CommonFilesMetadataBuilder {
final public class SingleDataSource extends CommonFilesMetadataBuilder {
private static final String WHERE_CLAUSE = "%s md5 in (select md5 from tsk_files where md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL) and data_source_obj_id=%s%s) GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; //NON-NLS
private final Long selectedDataSourceId;
@@ -35,10 +35,12 @@ final class SingleDataSource extends CommonFilesMetadataBuilder {
* once in the given data source
* @param dataSourceId data source id for which common files must appear at least once
* @param dataSourceIdMap a map of obj_id to datasource name
* @param filterByMediaMimeType match only on files whose mime types can be broadly categorized as media types
* @param filterByDocMimeType match only on files whose mime types can be broadly categorized as document types
* @param filterByMediaMimeType match only on files whose mime types can be
* broadly categorized as media types
* @param filterByDocMimeType match only on files whose mime types can be
* broadly categorized as document types
*/
SingleDataSource(Long dataSourceId, Map<Long, String> dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) {
public SingleDataSource(Long dataSourceId, Map<Long, String> dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) {
super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType);
this.selectedDataSourceId = dataSourceId;
this.dataSourceName = dataSourceIdMap.get(this.selectedDataSourceId);
Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java
+1 -1
View File
@@ -253,7 +253,7 @@ final public class FiltersPanel extends JPanel {
});
}
} catch (NoCurrentCaseException ex) {
logger.log(Level.WARNING, "Communications Visualization Tool opened with no open case.", ex);
logger.log(Level.INFO, "Filter update cancelled. Case is closed.");
} catch (TskCoreException tskCoreException) {
logger.log(Level.SEVERE, "There was a error loading the datasources for the case.", tskCoreException);
}
Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataResult.java
+1 -1
View File
@@ -63,7 +63,7 @@ public interface DataResult {
* Sets the descriptive text about the source of the nodes displayed in this
* result view component.
*
* @param description The text to display.
* @param pathText The text to display.
*/
public void setPath(String pathText);
Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java
+1 -1
View File
@@ -734,7 +734,7 @@ public class DataResultPanel extends javax.swing.JPanel implements DataResult, C
*
* @return True or false.
*
* @Deprecated This method has no valid use case.
* @deprecated This method has no valid use case.
*/
@Deprecated
@Override
Core/src/org/sleuthkit/autopsy/datamodel/AutopsyVisitableItem.java
+1 -1
View File
@@ -28,7 +28,7 @@ public interface AutopsyVisitableItem {
/**
* visitor pattern support
*
* @param v visitor
* @param visitor visitor
*
* @return visitor return value
*/
Core/src/org/sleuthkit/autopsy/datamodel/ContentNode.java
+1 -1
View File
@@ -40,7 +40,7 @@ abstract class ContentNode extends DisplayableItemNode {
/**
* Visitor pattern support.
*
* @param v visitor
* @param visitor visitor
*
* @return visitor's visit return value
*/
Core/src/org/sleuthkit/autopsy/healthmonitor/EnterpriseHealthMonitor.java
+418 -42
View File
@@ -28,8 +28,11 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.List;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -37,6 +40,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.Random;
import org.apache.commons.dbcp2.BasicDataSource;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
@@ -44,7 +48,6 @@ import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.core.UserPreferencesException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.coreutils.ThreadUtils;
import org.sleuthkit.datamodel.CaseDbConnectionInfo;
import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
@@ -64,8 +67,6 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
private final static Logger logger = Logger.getLogger(EnterpriseHealthMonitor.class.getName());
private final static String DATABASE_NAME = "EnterpriseHealthMonitor";
private final static String MODULE_NAME = "EnterpriseHealthMonitor";
private final static String IS_ENABLED_KEY = "is_enabled";
private final static long DATABASE_WRITE_INTERVAL = 60; // Minutes
public static final CaseDbSchemaVersionNumber CURRENT_DB_SCHEMA_VERSION
= new CaseDbSchemaVersionNumber(1, 0);
@@ -80,6 +81,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
private final Map<String, TimingInfo> timingInfoMap;
private static final int CONN_POOL_SIZE = 10;
private BasicDataSource connectionPool = null;
private CaseDbConnectionInfo connectionSettingsInUse = null;
private String hostName;
private EnterpriseHealthMonitor() throws HealthMonitorException {
@@ -100,22 +102,11 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
logger.log(Level.SEVERE, "Unable to look up host name - falling back to UUID " + hostName, ex);
}
// Read from module settings to determine if the module is enabled
if (ModuleSettings.settingExists(MODULE_NAME, IS_ENABLED_KEY)) {
if(ModuleSettings.getConfigSetting(MODULE_NAME, IS_ENABLED_KEY).equals("true")){
isEnabled.set(true);
try {
activateMonitor();
} catch (HealthMonitorException ex) {
// If we failed to activate it, then disable the monitor
logger.log(Level.SEVERE, "Health monitor activation failed - disabling health monitor");
setEnabled(false);
throw ex;
}
return;
}
}
isEnabled.set(false);
// Read from the database to determine if the module is enabled
updateFromGlobalEnabledStatus();
// Start the timer for database checks and writes
startTimer();
}
/**
@@ -137,10 +128,13 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
* out of the maps, and sets up the timer for writing to the database.
* @throws HealthMonitorException
*/
private synchronized void activateMonitor() throws HealthMonitorException {
private synchronized void activateMonitorLocally() throws HealthMonitorException {
logger.log(Level.INFO, "Activating Servies Health Monitor");
// Make sure there are no left over connections to an old database
shutdownConnections();
if (!UserPreferences.getIsMultiUserModeEnabled()) {
throw new HealthMonitorException("Multi user mode is not enabled - can not activate health monitor");
}
@@ -168,9 +162,6 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
// Clear out any old data
timingInfoMap.clear();
// Start the timer for database writes
startTimer();
}
/**
@@ -180,16 +171,13 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
* and shuts down the connection pool.
* @throws HealthMonitorException
*/
private synchronized void deactivateMonitor() throws HealthMonitorException {
private synchronized void deactivateMonitorLocally() throws HealthMonitorException {
logger.log(Level.INFO, "Deactivating Servies Health Monitor");
// Clear out the collected data
timingInfoMap.clear();
// Stop the timer
stopTimer();
// Shut down the connection pool
shutdownConnections();
}
@@ -202,7 +190,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
stopTimer();
healthMonitorOutputTimer = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().setNameFormat("health_monitor_timer").build());
healthMonitorOutputTimer.scheduleWithFixedDelay(new DatabaseWriteTask(), DATABASE_WRITE_INTERVAL, DATABASE_WRITE_INTERVAL, TimeUnit.MINUTES);
healthMonitorOutputTimer.scheduleWithFixedDelay(new PeriodicHealthMonitorTask(), DATABASE_WRITE_INTERVAL, DATABASE_WRITE_INTERVAL, TimeUnit.MINUTES);
}
/**
@@ -234,15 +222,18 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
}
if(enabled) {
getInstance().activateMonitor();
getInstance().activateMonitorLocally();
// If activateMonitor fails, we won't update either of these
ModuleSettings.setConfigSetting(MODULE_NAME, IS_ENABLED_KEY, "true");
// If activateMonitor fails, we won't update this
getInstance().setGlobalEnabledStatusInDB(true);
isEnabled.set(true);
} else {
ModuleSettings.setConfigSetting(MODULE_NAME, IS_ENABLED_KEY, "false");
if(isEnabled.get()) {
// If we were enabled before, set the global state to disabled
getInstance().setGlobalEnabledStatusInDB(false);
}
isEnabled.set(false);
getInstance().deactivateMonitor();
getInstance().deactivateMonitorLocally();
}
}
@@ -460,7 +451,6 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
String createCommand = "SELECT 1 AS result FROM pg_database WHERE datname='" + DATABASE_NAME + "'";
rs = statement.executeQuery(createCommand);
if(rs.next()) {
logger.log(Level.INFO, "Existing Enterprise Health Monitor database found");
return true;
}
} finally {
@@ -501,6 +491,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
private void setupConnectionPool() throws HealthMonitorException {
try {
CaseDbConnectionInfo db = UserPreferences.getDatabaseConnectionInfo();
connectionSettingsInUse = db;
connectionPool = new BasicDataSource();
connectionPool.setDriverClassName("org.postgresql.Driver");
@@ -598,6 +589,108 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
}
}
/**
* Return whether the health monitor is locally enabled.
* This does not query the database.
* @return true if it is enabled, false otherwise
*/
static boolean monitorIsEnabled() {
return isEnabled.get();
}
/**
* Check whether monitoring should be enabled from the monitor database
* and enable/disable as needed.
* @throws HealthMonitorException
*/
synchronized void updateFromGlobalEnabledStatus() throws HealthMonitorException {
boolean previouslyEnabled = monitorIsEnabled();
// We can't even check the database if multi user settings aren't enabled.
if (!UserPreferences.getIsMultiUserModeEnabled()) {
isEnabled.set(false);
if(previouslyEnabled) {
deactivateMonitorLocally();
}
return;
}
// If the health monitor database doesn't exist or if it is not initialized,
// then monitoring isn't enabled
if ((! databaseExists()) || (! databaseIsInitialized())) {
isEnabled.set(false);
if(previouslyEnabled) {
deactivateMonitorLocally();
}
return;
}
// If we're currently enabled, check whether the multiuser settings have changed.
// If they have, force a reset on the connection pool.
if(previouslyEnabled && (connectionSettingsInUse != null)) {
try {
CaseDbConnectionInfo currentSettings = UserPreferences.getDatabaseConnectionInfo();
if(! (connectionSettingsInUse.getUserName().equals(currentSettings.getUserName())
&& connectionSettingsInUse.getPassword().equals(currentSettings.getPassword())
&& connectionSettingsInUse.getPort().equals(currentSettings.getPort())
&& connectionSettingsInUse.getHost().equals(currentSettings.getHost()) )) {
shutdownConnections();
}
} catch (UserPreferencesException ex) {
throw new HealthMonitorException("Error reading database connection info", ex);
}
}
boolean currentlyEnabled = getGlobalEnabledStatusFromDB();
if( currentlyEnabled != previouslyEnabled) {
if( ! currentlyEnabled ) {
isEnabled.set(false);
deactivateMonitorLocally();
} else {
isEnabled.set(true);
activateMonitorLocally();
}
}
}
/**
* Read the enabled status from the database.
* Check that the health monitor database exists before calling this.
* @return true if the database is enabled, false otherwise
* @throws HealthMonitorException
*/
private boolean getGlobalEnabledStatusFromDB() throws HealthMonitorException {
try (Connection conn = connect();
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='MONITOR_ENABLED'")) {
if (resultSet.next()) {
return(resultSet.getBoolean("value"));
}
throw new HealthMonitorException("No enabled status found in database");
} catch (SQLException ex) {
throw new HealthMonitorException("Error initializing database", ex);
}
}
/**
* Set the global enabled status in the database.
* @throws HealthMonitorException
*/
private void setGlobalEnabledStatusInDB(boolean status) throws HealthMonitorException {
try (Connection conn = connect();
Statement statement = conn.createStatement();) {
statement.execute("UPDATE db_info SET value='" + status + "' WHERE name='MONITOR_ENABLED'");
} catch (SQLException ex) {
throw new HealthMonitorException("Error setting enabled status", ex);
}
}
/**
* Get the current schema version
* @return the current schema version
@@ -611,7 +704,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
ResultSet resultSet = null;
try (Statement statement = conn.createStatement()) {
int minorVersion = 0;
int minorVersion = 0;
int majorVersion = 0;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='SCHEMA_MINOR_VERSION'");
if (resultSet.next()) {
@@ -688,6 +781,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
statement.execute("INSERT INTO db_info (name, value) VALUES ('SCHEMA_VERSION', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('SCHEMA_MINOR_VERSION', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('MONITOR_ENABLED', 'true')");
conn.commit();
} catch (SQLException ex) {
@@ -708,20 +802,24 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
/**
* The task called by the ScheduledThreadPoolExecutor to handle
* the database writes.
* the database checks/writes.
*/
static final class DatabaseWriteTask implements Runnable {
static final class PeriodicHealthMonitorTask implements Runnable {
/**
* Write current metric data to the database
* Perform all periodic tasks:
* - Check if monitoring has been enabled / disabled in the database
* - Gather any additional metrics
* - Write current metric data to the database
*/
@Override
public void run() {
try {
getInstance().updateFromGlobalEnabledStatus();
getInstance().gatherTimerBasedMetrics();
getInstance().writeCurrentStateToDatabase();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error writing current metrics to database", ex); //NON-NLS
logger.log(Level.SEVERE, "Error performing periodic task", ex); //NON-NLS
}
}
}
@@ -734,12 +832,220 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
case CURRENT_CASE:
if ((null == evt.getNewValue()) && (evt.getOldValue() instanceof Case)) {
// When a case is closed, write the current metrics to the database
healthMonitorExecutor.submit(new EnterpriseHealthMonitor.DatabaseWriteTask());
healthMonitorExecutor.submit(new EnterpriseHealthMonitor.PeriodicHealthMonitorTask());
}
break;
}
}
/**
* Debugging method to generate sample data for the database.
* It will delete all current timing data and replace it with randomly generated values.
* If there is more than one node, the second node's times will trend upwards.
*/
void populateDatabaseWithSampleData(int nDays, int nNodes, boolean createVerificationData) throws HealthMonitorException {
if(! isEnabled.get()) {
throw new HealthMonitorException("Can't populate database - monitor not enabled");
}
// Get the database lock
CoordinationService.Lock lock = getSharedDbLock();
if(lock == null) {
throw new HealthMonitorException("Error getting database lock");
}
String[] metricNames = {"Disk Reads: Hash calculation", "Database: getImages query", "Solr: Index chunk", "Solr: Connectivity check"}; // NON-NLS
Random rand = new Random();
long maxTimestamp = System.currentTimeMillis();
long millisPerHour = 1000 * 60 * 60;
long minTimestamp = maxTimestamp - (nDays * (millisPerHour * 24));
Connection conn = null;
try {
conn = connect();
if(conn == null) {
throw new HealthMonitorException("Error getting database connection");
}
try (Statement statement = conn.createStatement()) {
statement.execute("DELETE FROM timing_data"); // NON-NLS
} catch (SQLException ex) {
logger.log(Level.SEVERE, "Error clearing timing data", ex);
return;
}
// Add timing metrics to the database
String addTimingInfoSql = "INSERT INTO timing_data (name, host, timestamp, count, average, max, min) VALUES (?, ?, ?, ?, ?, ?, ?)";
try (PreparedStatement statement = conn.prepareStatement(addTimingInfoSql)) {
for(String metricName:metricNames) {
long baseIndex = rand.nextInt(900) + 100;
int multiplier = rand.nextInt(5);
long minIndexTimeNanos;
switch(multiplier) {
case 0:
minIndexTimeNanos = baseIndex;
break;
case 1:
minIndexTimeNanos = baseIndex * 1000;
break;
default:
minIndexTimeNanos = baseIndex * 1000 * 1000;
break;
}
long maxIndexTimeOverMin = minIndexTimeNanos * 3;
for(int node = 0;node < nNodes; node++) {
String host = "testHost" + node; // NON-NLS
double count = 0;
double maxCount = nDays * 24 + 1;
// Record data every hour, with a small amount of randomness about when it starts
for(long timestamp = minTimestamp + rand.nextInt(1000 * 60 * 55);timestamp < maxTimestamp;timestamp += millisPerHour) {
double aveTime;
// This creates data that increases in the last couple of days of the simulated
// collection
count++;
double slowNodeMultiplier = 1.0;
if((maxCount - count) <= 3 * 24) {
slowNodeMultiplier += (3 - (maxCount - count) / 24) * 0.33;
}
if( ! createVerificationData ) {
// Try to make a reasonable sample data set, with most points in a small range
// but some higher and lower
int outlierVal = rand.nextInt(30);
long randVal = rand.nextLong();
if(randVal < 0) {
randVal *= -1;
}
if(outlierVal < 2){
aveTime = minIndexTimeNanos + maxIndexTimeOverMin + randVal % maxIndexTimeOverMin;
} else if(outlierVal == 2){
aveTime = (minIndexTimeNanos / 2) + randVal % (minIndexTimeNanos / 2);
} else if(outlierVal < 17) {
aveTime = minIndexTimeNanos + randVal % (maxIndexTimeOverMin / 2);
} else {
aveTime = minIndexTimeNanos + randVal % maxIndexTimeOverMin;
}
if(node == 1) {
aveTime = aveTime * slowNodeMultiplier;
}
} else {
// Create a data set strictly for testing that the display is working
// correctly. The average time will equal the day of the month from
// the timestamp (in milliseconds)
Calendar thisDate = new GregorianCalendar();
thisDate.setTimeInMillis(timestamp);
int day = thisDate.get(Calendar.DAY_OF_MONTH);
aveTime = day * 1000000;
}
statement.setString(1, metricName);
statement.setString(2, host);
statement.setLong(3, timestamp);
statement.setLong(4, 0);
statement.setDouble(5, aveTime / 1000000);
statement.setDouble(6, 0);
statement.setDouble(7, 0);
statement.execute();
}
}
}
} catch (SQLException ex) {
throw new HealthMonitorException("Error saving metric data to database", ex);
}
} finally {
try {
if(conn != null) {
conn.close();
}
} catch (SQLException ex) {
logger.log(Level.SEVERE, "Error closing Connection.", ex);
}
try {
lock.release();
} catch (CoordinationService.CoordinationServiceException ex) {
throw new HealthMonitorException("Error releasing database lock", ex);
}
}
}
/**
* Get timing metrics currently stored in the database.
* @param timeRange Maximum age for returned metrics (in milliseconds)
* @return A map with metric name mapped to a list of data
* @throws HealthMonitorException
*/
Map<String, List<DatabaseTimingResult>> getTimingMetricsFromDatabase(long timeRange) throws HealthMonitorException {
// Make sure the monitor is enabled. It could theoretically get disabled after this
// check but it doesn't seem worth holding a lock to ensure that it doesn't since that
// may slow down ingest.
if(! isEnabled.get()) {
throw new HealthMonitorException("Health Monitor is not enabled");
}
// Calculate the smallest timestamp we should return
long minimumTimestamp = System.currentTimeMillis() - timeRange;
try (CoordinationService.Lock lock = getSharedDbLock()) {
if(lock == null) {
throw new HealthMonitorException("Error getting database lock");
}
Connection conn = connect();
if(conn == null) {
throw new HealthMonitorException("Error getting database connection");
}
Map<String, List<DatabaseTimingResult>> resultMap = new HashMap<>();
try (Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT * FROM timing_data WHERE timestamp > " + minimumTimestamp)) {
while (resultSet.next()) {
String name = resultSet.getString("name");
DatabaseTimingResult timingResult = new DatabaseTimingResult(resultSet);
if(resultMap.containsKey(name)) {
resultMap.get(name).add(timingResult);
} else {
List<DatabaseTimingResult> resultList = new ArrayList<>();
resultList.add(timingResult);
resultMap.put(name, resultList);
}
}
return resultMap;
} catch (SQLException ex) {
throw new HealthMonitorException("Error reading timing metrics from database", ex);
} finally {
try {
conn.close();
} catch (SQLException ex) {
logger.log(Level.SEVERE, "Error closing Connection.", ex);
}
}
} catch (CoordinationService.CoordinationServiceException ex) {
throw new HealthMonitorException("Error getting database lock", ex);
}
}
/**
* Get an exclusive lock for the health monitor database.
* Acquire this before creating, initializing, or updating the database schema.
@@ -856,4 +1162,74 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
return count;
}
}
/**
* Class for retrieving timing metrics from the database to display to the user.
* All times will be in milliseconds.
*/
static class DatabaseTimingResult {
private final long timestamp; // Time the metric was recorded
private final String hostname; // Host that recorded the metric
private final long count; // Number of metrics collected
private final double average; // Average of the durations collected (milliseconds)
private final double max; // Maximum value found (milliseconds)
private final double min; // Minimum value found (milliseconds)
DatabaseTimingResult(ResultSet resultSet) throws SQLException {
this.timestamp = resultSet.getLong("timestamp");
this.hostname = resultSet.getString("host");
this.count = resultSet.getLong("count");
this.average = resultSet.getDouble("average");
this.max = resultSet.getDouble("max");
this.min = resultSet.getDouble("min");
}
/**
* Get the timestamp for when the metric was recorded
* @return
*/
long getTimestamp() {
return timestamp;
}
/**
* Get the average duration
* @return average duration (milliseconds)
*/
double getAverage() {
return average;
}
/**
* Get the maximum duration
* @return maximum duration (milliseconds)
*/
double getMax() {
return max;
}
/**
* Get the minimum duration
* @return minimum duration (milliseconds)
*/
double getMin() {
return min;
}
/**
* Get the total number of metrics collected
* @return number of metrics collected
*/
long getCount() {
return count;
}
/**
* Get the name of the host that recorded this metric
* @return the host
*/
String getHostName() {
return hostname;
}
}
}
Core/src/org/sleuthkit/autopsy/healthmonitor/HealthMonitorDashboard.java
+513
View File
@@ -0,0 +1,513 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.healthmonitor;
import java.awt.Container;
import java.awt.Cursor;
import java.awt.Dimension;
import java.util.Set;
import java.util.HashSet;
import java.util.HashMap;
import java.util.Arrays;
import java.util.List;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.File;
import javax.swing.Box;
import javax.swing.JButton;
import javax.swing.JDialog;
import javax.swing.JComboBox;
import javax.swing.JSeparator;
import javax.swing.JCheckBox;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.BorderFactory;
import java.util.Map;
import javax.swing.BoxLayout;
import java.awt.GridLayout;
import java.nio.file.Paths;
import java.util.logging.Level;
import java.util.stream.Collectors;
import org.openide.modules.Places;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
/**
* Dashboard for viewing metrics and controlling the health monitor.
*/
public class HealthMonitorDashboard {
private final static Logger logger = Logger.getLogger(HealthMonitorDashboard.class.getName());
private final static String ADMIN_ACCESS_FILE_NAME = "adminAccess"; // NON-NLS
private final static String ADMIN_ACCESS_FILE_PATH = Paths.get(Places.getUserDirectory().getAbsolutePath(), ADMIN_ACCESS_FILE_NAME).toString();
Map<String, List<EnterpriseHealthMonitor.DatabaseTimingResult>> timingData;
private JComboBox<String> dateComboBox = null;
private JComboBox<String> hostComboBox = null;
private JCheckBox hostCheckBox = null;
private JCheckBox showTrendLineCheckBox = null;
private JCheckBox skipOutliersCheckBox = null;
private JPanel graphPanel = null;
private JDialog dialog = null;
private final Container parentWindow;
/**
* Create an instance of the dashboard.
* Call display() after creation to show the dashboard.
* @param parent The parent container (for centering the UI)
*/
public HealthMonitorDashboard(Container parent) {
timingData = new HashMap<>();
parentWindow = parent;
}
/**
* Display the dashboard.
*/
@NbBundle.Messages({"HealthMonitorDashboard.display.errorCreatingDashboard=Error creating health monitor dashboard",
"HealthMonitorDashboard.display.dashboardTitle=Enterprise Health Monitor"})
public void display() {
// Update the enabled status and get the timing data, then create all
// the sub panels.
JPanel timingPanel;
JPanel adminPanel;
try {
updateData();
timingPanel = createTimingPanel();
adminPanel = createAdminPanel();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error creating panels for health monitor dashboard", ex);
MessageNotifyUtil.Message.error(Bundle.HealthMonitorDashboard_display_errorCreatingDashboard());
return;
}
// Create the main panel for the dialog
JPanel mainPanel = new JPanel();
mainPanel.setLayout(new BoxLayout(mainPanel, BoxLayout.Y_AXIS));
// Add the timing panel
mainPanel.add(timingPanel);
// Add the admin panel if the admin file is present
File adminFile = new File(ADMIN_ACCESS_FILE_PATH);
if(adminFile.exists()) {
mainPanel.add(adminPanel);
}
// Create and show the dialog
dialog = new JDialog();
dialog.setTitle(Bundle.HealthMonitorDashboard_display_dashboardTitle());
dialog.add(mainPanel);
dialog.pack();
dialog.setLocationRelativeTo(parentWindow);
dialog.setVisible(true);
}
/**
* Delete the current dialog and create a new one. This should only be
* called after enabling or disabling the health monitor.
*/
private void redisplay() {
if (dialog != null) {
dialog.setVisible(false);
dialog.dispose();
}
display();
}
/**
* Check the monitor enabled status and, if enabled, get the timing data.
* @throws HealthMonitorException
*/
private void updateData() throws HealthMonitorException {
// Update the monitor status
EnterpriseHealthMonitor.getInstance().updateFromGlobalEnabledStatus();
if(EnterpriseHealthMonitor.monitorIsEnabled()) {
// Get a copy of the timing data from the database
timingData = EnterpriseHealthMonitor.getInstance().getTimingMetricsFromDatabase(DateRange.getMaximumTimestampRange());
}
}
/**
* Create the panel holding the timing graphs and the controls for them.
* @return The timing panel
* @throws HealthMonitorException
*/
@NbBundle.Messages({"HealthMonitorDashboard.createTimingPanel.noData=No data to display - monitor is not enabled",
"HealthMonitorDashboard.createTimingPanel.timingMetricsTitle=Timing Metrics"})
private JPanel createTimingPanel() throws HealthMonitorException {
// If the monitor isn't enabled, just add a message
if(! EnterpriseHealthMonitor.monitorIsEnabled()) {
//timingMetricPanel.setPreferredSize(new Dimension(400,100));
JPanel emptyTimingMetricPanel = new JPanel();
emptyTimingMetricPanel.add(new JLabel(Bundle.HealthMonitorDashboard_createTimingPanel_timingMetricsTitle()));
emptyTimingMetricPanel.add(new JLabel(" "));
emptyTimingMetricPanel.add(new JLabel(Bundle.HealthMonitorDashboard_createTimingPanel_noData()));
return emptyTimingMetricPanel;
}
JPanel timingMetricPanel = new JPanel();
timingMetricPanel.setLayout(new BoxLayout(timingMetricPanel, BoxLayout.PAGE_AXIS));
timingMetricPanel.setBorder(BorderFactory.createEtchedBorder());
// Add title
JLabel timingMetricTitle = new JLabel(Bundle.HealthMonitorDashboard_createTimingPanel_timingMetricsTitle());
timingMetricPanel.add(timingMetricTitle);
timingMetricPanel.add(new JSeparator());
// Add the controls
timingMetricPanel.add(createTimingControlPanel());
timingMetricPanel.add(new JSeparator());
// Create panel to hold graphs
graphPanel = new JPanel();
graphPanel.setLayout(new GridLayout(0,2));
// Update the graph panel, put it in a scroll pane, and add to the timing metric panel
updateTimingMetricGraphs();
JScrollPane scrollPane = new JScrollPane(graphPanel, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_NEVER);
timingMetricPanel.add(scrollPane);
timingMetricPanel.revalidate();
timingMetricPanel.repaint();
return timingMetricPanel;
}
/**
* Create the panel with combo boxes for date range and host.
* @return the control panel
*/
@NbBundle.Messages({"HealthMonitorDashboard.createTimingControlPanel.filterByHost=Filter by host",
"HealthMonitorDashboard.createTimingControlPanel.maxDays=Max days to display",
"HealthMonitorDashboard.createTimingControlPanel.skipOutliers=Do not plot outliers",
"HealthMonitorDashboard.createTimingControlPanel.showTrendLine=Show trend line"})
private JPanel createTimingControlPanel() {
JPanel timingControlPanel = new JPanel();
// If the monitor is not enabled, don't add any components
if(! EnterpriseHealthMonitor.monitorIsEnabled()) {
return timingControlPanel;
}
// Create the combo box for selecting how much data to display
String[] dateOptionStrings = Arrays.stream(DateRange.values()).map(e -> e.getLabel()).toArray(String[]::new);
dateComboBox = new JComboBox<>(dateOptionStrings);
dateComboBox.setSelectedItem(DateRange.ONE_DAY.getLabel());
// Set up the listener on the date combo box
dateComboBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error updating timing metric panel", ex);
}
}
});
// Create an array of host names
Set<String> hostNameSet = new HashSet<>();
for(String metricType:timingData.keySet()) {
for(EnterpriseHealthMonitor.DatabaseTimingResult result: timingData.get(metricType)) {
hostNameSet.add(result.getHostName());
}
}
// Load the host names into the combo box
hostComboBox = new JComboBox<>(hostNameSet.toArray(new String[hostNameSet.size()]));
// Set up the listener on the combo box
hostComboBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
if((hostCheckBox != null) && hostCheckBox.isSelected()) {
updateTimingMetricGraphs();
}
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Create the host checkbox
hostCheckBox = new JCheckBox(Bundle.HealthMonitorDashboard_createTimingControlPanel_filterByHost());
hostCheckBox.setSelected(false);
hostComboBox.setEnabled(false);
// Set up the listener on the checkbox
hostCheckBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
hostComboBox.setEnabled(hostCheckBox.isSelected());
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Create the checkbox for showing the trend line
showTrendLineCheckBox = new JCheckBox(Bundle.HealthMonitorDashboard_createTimingControlPanel_showTrendLine());
showTrendLineCheckBox.setSelected(true);
// Set up the listener on the checkbox
showTrendLineCheckBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Create the checkbox for omitting outliers
skipOutliersCheckBox = new JCheckBox(Bundle.HealthMonitorDashboard_createTimingControlPanel_skipOutliers());
skipOutliersCheckBox.setSelected(false);
// Set up the listener on the checkbox
skipOutliersCheckBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Add the date range combo box and label to the panel
timingControlPanel.add(new JLabel(Bundle.HealthMonitorDashboard_createTimingControlPanel_maxDays()));
timingControlPanel.add(dateComboBox);
// Put some space between the elements
timingControlPanel.add(Box.createHorizontalStrut(100));
// Add the host combo box and checkbox to the panel
timingControlPanel.add(hostCheckBox);
timingControlPanel.add(hostComboBox);
// Put some space between the elements
timingControlPanel.add(Box.createHorizontalStrut(100));
// Add the skip outliers checkbox
timingControlPanel.add(this.showTrendLineCheckBox);
// Put some space between the elements
timingControlPanel.add(Box.createHorizontalStrut(100));
// Add the skip outliers checkbox
timingControlPanel.add(this.skipOutliersCheckBox);
return timingControlPanel;
}
/**
* Update the timing graphs.
* @throws HealthMonitorException
*/
@NbBundle.Messages({"HealthMonitorDashboard.updateTimingMetricGraphs.noData=No data to display"})
private void updateTimingMetricGraphs() throws HealthMonitorException {
// Clear out any old graphs
graphPanel.removeAll();
if(timingData.keySet().isEmpty()) {
// There are no timing metrics in the database
graphPanel.add(new JLabel(Bundle.HealthMonitorDashboard_updateTimingMetricGraphs_noData()));
return;
}
for(String metricName:timingData.keySet()) {
// If necessary, trim down the list of results to fit the selected time range
List<EnterpriseHealthMonitor.DatabaseTimingResult> intermediateTimingDataForDisplay;
if(dateComboBox.getSelectedItem() != null) {
DateRange selectedDateRange = DateRange.fromLabel(dateComboBox.getSelectedItem().toString());
long threshold = System.currentTimeMillis() - selectedDateRange.getTimestampRange();
intermediateTimingDataForDisplay = timingData.get(metricName).stream()
.filter(t -> t.getTimestamp() > threshold)
.collect(Collectors.toList());
} else {
intermediateTimingDataForDisplay = timingData.get(metricName);
}
// Get the name of the selected host, if there is one.
// The graph always uses the data from all hosts to generate the x and y scales
// so we don't filter anything out here.
String hostToDisplay = null;
if(hostCheckBox.isSelected() && (hostComboBox.getSelectedItem() != null)) {
hostToDisplay = hostComboBox.getSelectedItem().toString();
}
// Generate the graph
TimingMetricGraphPanel singleTimingGraphPanel = new TimingMetricGraphPanel(intermediateTimingDataForDisplay,
hostToDisplay, true, metricName, skipOutliersCheckBox.isSelected(), showTrendLineCheckBox.isSelected());
singleTimingGraphPanel.setPreferredSize(new Dimension(700,200));
graphPanel.add(singleTimingGraphPanel);
}
graphPanel.revalidate();
graphPanel.repaint();
}
/**
* Create the admin panel.
* This allows the health monitor to be enabled and disabled.
* @return
*/
@NbBundle.Messages({"HealthMonitorDashboard.createAdminPanel.enableButton=Enable monitor",
"HealthMonitorDashboard.createAdminPanel.disableButton=Disable monitor"})
private JPanel createAdminPanel() {
JPanel adminPanel = new JPanel();
adminPanel.setBorder(BorderFactory.createEtchedBorder());
// Create the buttons for enabling/disabling the monitor
JButton enableButton = new JButton(Bundle.HealthMonitorDashboard_createAdminPanel_enableButton());
JButton disableButton = new JButton(Bundle.HealthMonitorDashboard_createAdminPanel_disableButton());
boolean isEnabled = EnterpriseHealthMonitor.monitorIsEnabled();
enableButton.setEnabled(! isEnabled);
disableButton.setEnabled(isEnabled);
// Set up a listener on the enable button
enableButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
EnterpriseHealthMonitor.setEnabled(true);
redisplay();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error enabling monitoring", ex);
} finally {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
}
}
});
// Set up a listener on the disable button
disableButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
EnterpriseHealthMonitor.setEnabled(false);
redisplay();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error disabling monitoring", ex);
} finally {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
}
}
});
// Add the buttons
adminPanel.add(enableButton);
adminPanel.add(Box.createHorizontalStrut(25));
adminPanel.add(disableButton);
return adminPanel;
}
/**
* Possible date ranges for the metrics in the UI
*/
@NbBundle.Messages({"HealthMonitorDashboard.DateRange.oneMonth=One month",
"HealthMonitorDashboard.DateRange.twoWeeks=Two weeks",
"HealthMonitorDashboard.DateRange.oneWeek=One week",
"HealthMonitorDashboard.DateRange.oneDay=One day"})
private enum DateRange {
ONE_DAY(Bundle.HealthMonitorDashboard_DateRange_oneDay(), 1),
ONE_WEEK(Bundle.HealthMonitorDashboard_DateRange_oneWeek(), 7),
TWO_WEEKS(Bundle.HealthMonitorDashboard_DateRange_twoWeeks(), 14),
ONE_MONTH(Bundle.HealthMonitorDashboard_DateRange_oneMonth(), 31);
private final String label;
private final long numberOfDays;
private static final long MILLISECONDS_PER_DAY = 1000 * 60 * 60 * 24;
DateRange(String label, long numberOfDays) {
this.label = label;
this.numberOfDays = numberOfDays;
}
/**
* Get the name for display in the UI
* @return the name
*/
String getLabel() {
return label;
}
/**
* Get the number of milliseconds represented by this date range.
* Compare the timestamps to ((current time in millis) - (this value)) to
* determine if they are in the range
* @return the time range in milliseconds
*/
long getTimestampRange() {
if (numberOfDays > 0) {
return numberOfDays * MILLISECONDS_PER_DAY;
} else {
return Long.MAX_VALUE;
}
}
/**
* Get the maximum range for this enum.
* This should be used for querying the database for the timing metrics to display.
* @return the maximum range in milliseconds
*/
static long getMaximumTimestampRange() {
long maxRange = Long.MIN_VALUE;
for (DateRange dateRange : DateRange.values()) {
if (dateRange.getTimestampRange() > maxRange) {
maxRange = dateRange.getTimestampRange();
}
}
return maxRange;
}
static DateRange fromLabel(String text) {
for (DateRange dateRange : DateRange.values()) {
if (dateRange.label.equalsIgnoreCase(text)) {
return dateRange;
}
}
return ONE_DAY; // If the comparison failed, return a default
}
}
}
Core/src/org/sleuthkit/autopsy/healthmonitor/TimingMetricGraphPanel.java
+510
View File
@@ -0,0 +1,510 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.healthmonitor;
import java.awt.BasicStroke;
import java.awt.Color;
import java.awt.FontMetrics;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.Point;
import java.awt.RenderingHints;
import java.awt.Stroke;
import java.util.Collections;
import java.util.stream.Collectors;
import java.util.Comparator;
import java.util.ArrayList;
import java.util.List;
import java.util.Calendar;
import java.util.GregorianCalendar;
import javax.swing.JPanel;
import org.sleuthkit.autopsy.coreutils.Logger;
import java.util.logging.Level;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.healthmonitor.EnterpriseHealthMonitor.DatabaseTimingResult;
/**
* Creates a graph of the given timing metric data
*/
class TimingMetricGraphPanel extends JPanel {
private final static Logger logger = Logger.getLogger(TimingMetricGraphPanel.class.getName());
private final int padding = 25;
private final int labelPadding = 25;
private final Color lineColor = new Color(0x12, 0x20, 0xdb, 180);
private final Color gridColor = new Color(200, 200, 200, 200);
private final Color trendLineColor = new Color(150, 10, 10, 200);
private static final Stroke GRAPH_STROKE = new BasicStroke(2f);
private static final Stroke NARROW_STROKE = new BasicStroke(1f);
private final int pointWidth = 4;
private final int numberYDivisions = 10;
private List<DatabaseTimingResult> timingResults;
private final String metricName;
private final boolean doLineGraph;
private final boolean skipOutliers;
private final boolean showTrendLine;
private String yUnitString;
private TrendLine trendLine;
private final long MILLISECONDS_PER_DAY = 1000 * 60 * 60 * 24;
private final long NANOSECONDS_PER_MILLISECOND = 1000 * 1000;
private long maxTimestamp;
private long minTimestamp;
private double maxMetricTime;
private double minMetricTime;
TimingMetricGraphPanel(List<DatabaseTimingResult> timingResultsFull,
String hostName, boolean doLineGraph, String metricName, boolean skipOutliers, boolean showTrendLine) {
this.doLineGraph = doLineGraph;
this.skipOutliers = skipOutliers;
this.showTrendLine = showTrendLine;
this.metricName = metricName;
if(hostName == null || hostName.isEmpty()) {
timingResults = timingResultsFull;
} else {
timingResults = timingResultsFull.stream()
.filter(t -> t.getHostName().equals(hostName))
.collect(Collectors.toList());
}
if(showTrendLine) {
try {
trendLine = new TrendLine(timingResults);
} catch (HealthMonitorException ex) {
// Log it, set trendLine to null and continue on
logger.log(Level.WARNING, "Can not generate a trend line on empty data set");
trendLine = null;
}
}
// Calculate these using the full data set, to make it easier to compare the results for
// individual hosts. Calculate the average at the same time.
maxMetricTime = Double.MIN_VALUE;
minMetricTime = Double.MAX_VALUE;
maxTimestamp = Long.MIN_VALUE;
minTimestamp = Long.MAX_VALUE;
double averageMetricTime = 0.0;
for (DatabaseTimingResult result : timingResultsFull) {
maxMetricTime = Math.max(maxMetricTime, result.getAverage());
minMetricTime = Math.min(minMetricTime, result.getAverage());
maxTimestamp = Math.max(maxTimestamp, result.getTimestamp());
minTimestamp = Math.min(minTimestamp, result.getTimestamp());
averageMetricTime += result.getAverage();
}
averageMetricTime = averageMetricTime / timingResultsFull.size();
// If we're omitting outliers, we may use a different maxMetricTime.
// If the max time is reasonably close to the average, do nothing
if (this.skipOutliers && (maxMetricTime > (averageMetricTime * 5))) {
// Calculate the standard deviation
double intermediateValue = 0.0;
for (DatabaseTimingResult result : timingResultsFull) {
double diff = result.getAverage() - averageMetricTime;
intermediateValue += diff * diff;
}
double standardDeviation = Math.sqrt(intermediateValue / timingResultsFull.size());
maxMetricTime = averageMetricTime + standardDeviation;
}
}
/**
* Setup of the graphics panel:
* Origin (0,0) is at the top left corner
*
* Horizontally (from the left): (padding)(label padding)(the graph)(padding)
* For plotting data on the x-axis, we scale it to the size of the graph and then add the padding and label padding
*
* Vertically (from the top): (padding)(the graph)(label padding)(padding)
* For plotting data on the y-axis, we subtract from the max value in the graph and then scale to the size of the graph
* @param g
*/
@NbBundle.Messages({"TimingMetricGraphPanel.paintComponent.nanoseconds=nanoseconds",
"TimingMetricGraphPanel.paintComponent.microseconds=microseconds",
"TimingMetricGraphPanel.paintComponent.milliseconds=milliseconds",
"TimingMetricGraphPanel.paintComponent.seconds=seconds",
"TimingMetricGraphPanel.paintComponent.minutes=minutes",
"TimingMetricGraphPanel.paintComponent.hours=hours",
"TimingMetricGraphPanel.paintComponent.displayingTime=displaying time in "})
@Override
protected void paintComponent(Graphics g) {
super.paintComponent(g);
Graphics2D g2 = (Graphics2D) g;
g2.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
// Get the max and min timestamps to create the x-axis.
// We add a small buffer to each side so the data won't overwrite the axes.
double maxValueOnXAxis = maxTimestamp + TimeUnit.HOURS.toMillis(2); // Two hour buffer
double minValueOnXAxis = minTimestamp - TimeUnit.HOURS.toMillis(2); // Two hour buffer
// Get the max and min times to create the y-axis
// We add a small buffer to each side so the data won't overwrite the axes.
double maxValueOnYAxis = maxMetricTime;
double minValueOnYAxis = minMetricTime;
minValueOnYAxis = Math.max(0, minValueOnYAxis - (maxValueOnYAxis * 0.1));
maxValueOnYAxis = maxValueOnYAxis * 1.1;
// The graph itself has the following corners:
// (padding + label padding, padding + font height) -> top left
// (padding + label padding, getHeight() - label padding - padding) -> bottom left
// (getWidth() - padding, padding + font height) -> top right
// (padding + label padding, getHeight() - label padding - padding) -> bottom right
int leftGraphPadding = padding + labelPadding;
int rightGraphPadding = padding;
int topGraphPadding = padding + g2.getFontMetrics().getHeight();
int bottomGraphPadding = labelPadding;
// Calculate the scale for each axis.
// The size of the graph area is the width/height of the panel minus any padding.
// The scale is calculated based on this size of the graph compared to the data range.
// For example:
// getWidth() = 575 => graph width = 500
// If our max x value to plot is 10000 and our min is 0, then the xScale would be 0.05 - i.e.,
// our original x values will be multipled by 0.05 to translate them to an x-coordinate in the
// graph (plus the padding)
int graphWidth = getWidth() - leftGraphPadding - rightGraphPadding;
int graphHeight = getHeight() - topGraphPadding - bottomGraphPadding;
double xScale = ((double) graphWidth) / (maxValueOnXAxis - minValueOnXAxis);
double yScale = ((double) graphHeight) / (maxValueOnYAxis - minValueOnYAxis);
// Check if we should use a scale other than milliseconds
// The idea here is to pick the scale that would most commonly be used to
// represent the middle of our data. For example, if the middle of the graph
// would be 45,000,000 nanoseconds, then we would use milliseconds for the
// y-axis.
long middleOfGraphNano = (long)((minValueOnYAxis + (maxValueOnYAxis - minValueOnYAxis) / 2.0) * NANOSECONDS_PER_MILLISECOND);
double yLabelScale;
if(middleOfGraphNano < TimeUnit.MICROSECONDS.toNanos(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_nanoseconds();
yLabelScale = TimeUnit.MILLISECONDS.toNanos(1);
} else if (TimeUnit.NANOSECONDS.toMicros(middleOfGraphNano) < TimeUnit.MILLISECONDS.toMicros(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_microseconds();
yLabelScale = TimeUnit.MILLISECONDS.toMicros(1);
} else if (TimeUnit.NANOSECONDS.toMillis(middleOfGraphNano) < TimeUnit.SECONDS.toMillis(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_milliseconds();
yLabelScale = 1;
} else if (TimeUnit.NANOSECONDS.toSeconds(middleOfGraphNano) < TimeUnit.MINUTES.toSeconds(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_seconds();
yLabelScale = 1.0 / TimeUnit.SECONDS.toMillis(1);
} else if (TimeUnit.NANOSECONDS.toMinutes(middleOfGraphNano) < TimeUnit.HOURS.toMinutes(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_minutes();
yLabelScale = 1.0 / (TimeUnit.MINUTES.toMillis(1));
} else {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_hours();
yLabelScale = 1.0 / (TimeUnit.HOURS.toMillis(1));
}
// Draw white background
g2.setColor(Color.WHITE);
g2.fillRect(leftGraphPadding, topGraphPadding, graphWidth, graphHeight);
// Create hatch marks and grid lines for y axis.
int labelWidth;
int positionForMetricNameLabel = 0;
for (int i = 0; i < numberYDivisions + 1; i++) {
int x0 = leftGraphPadding;
int x1 = pointWidth + leftGraphPadding;
int y0 = getHeight() - ((i * graphHeight) / numberYDivisions + bottomGraphPadding);
int y1 = y0;
if ( ! timingResults.isEmpty()) {
// Draw the grid line
g2.setColor(gridColor);
g2.drawLine(leftGraphPadding + 1 + pointWidth, y0, getWidth() - rightGraphPadding, y1);
// Create the label
g2.setColor(Color.BLACK);
double yValue = minValueOnYAxis + ((maxValueOnYAxis - minValueOnYAxis) * ((i * 1.0) / numberYDivisions));
String yLabel = Double.toString(((int) (yValue * 100 * yLabelScale)) / 100.0);
FontMetrics fontMetrics = g2.getFontMetrics();
labelWidth = fontMetrics.stringWidth(yLabel);
g2.drawString(yLabel, x0 - labelWidth - 5, y0 + (fontMetrics.getHeight() / 2) - 3);
// The nicest looking alignment for this label seems to be left-aligned with the top
// y-axis label. Save this position to be used to write the label later.
if (i == numberYDivisions) {
positionForMetricNameLabel = x0 - labelWidth - 5;
}
}
// Draw the small hatch mark
g2.setColor(Color.BLACK);
g2.drawLine(x0, y0, x1, y1);
}
// On the x-axis, the farthest right grid line should represent midnight preceding the last recorded value
Calendar maxDate = new GregorianCalendar();
maxDate.setTimeInMillis(maxTimestamp);
maxDate.set(Calendar.HOUR_OF_DAY, 0);
maxDate.set(Calendar.MINUTE, 0);
maxDate.set(Calendar.SECOND, 0);
maxDate.set(Calendar.MILLISECOND, 0);
long maxMidnightInMillis = maxDate.getTimeInMillis();
// We don't want to display more than 20 grid lines. If we have more
// data then that, put multiple days within one division
long totalDays = (maxMidnightInMillis - (long)minValueOnXAxis) / MILLISECONDS_PER_DAY;
long daysPerDivision;
if(totalDays <= 20) {
daysPerDivision = 1;
} else {
daysPerDivision = (totalDays / 20);
if((totalDays % 20) != 0) {
daysPerDivision++;
}
}
// Draw the vertical grid lines and labels
// The vertical grid lines will be at midnight, and display the date underneath them
// At present we use GMT because of some complications with daylight savings time.
for (long currentDivision = maxMidnightInMillis; currentDivision >= minValueOnXAxis; currentDivision -= MILLISECONDS_PER_DAY * daysPerDivision) {
int x0 = (int) ((currentDivision - minValueOnXAxis) * xScale + leftGraphPadding);
int x1 = x0;
int y0 = getHeight() - bottomGraphPadding;
int y1 = y0 - pointWidth;
// Draw the light grey grid line
g2.setColor(gridColor);
g2.drawLine(x0, getHeight() - bottomGraphPadding - 1 - pointWidth, x1, topGraphPadding);
// Draw the hatch mark
g2.setColor(Color.BLACK);
g2.drawLine(x0, y0, x1, y1);
// Draw the label
Calendar thisDate = new GregorianCalendar();
thisDate.setTimeZone(TimeZone.getTimeZone("GMT")); // Stick with GMT to avoid daylight savings issues
thisDate.setTimeInMillis(currentDivision);
int month = thisDate.get(Calendar.MONTH) + 1;
int day = thisDate.get(Calendar.DAY_OF_MONTH);
String xLabel = month + "/" + day;
FontMetrics metrics = g2.getFontMetrics();
labelWidth = metrics.stringWidth(xLabel);
g2.drawString(xLabel, x0 - labelWidth / 2, y0 + metrics.getHeight() + 3);
}
// Create x and y axes
g2.setColor(Color.BLACK);
g2.drawLine(leftGraphPadding, getHeight() - bottomGraphPadding, leftGraphPadding, topGraphPadding);
g2.drawLine(leftGraphPadding, getHeight() - bottomGraphPadding, getWidth() - rightGraphPadding, getHeight() - bottomGraphPadding);
// Create the points to plot
List<Point> graphPoints = new ArrayList<>();
for (int i = 0; i < timingResults.size(); i++) {
double metricTime = timingResults.get(i).getAverage();
int x1 = (int) ((timingResults.get(i).getTimestamp() - minValueOnXAxis) * xScale + leftGraphPadding);
int y1 = (int) ((maxValueOnYAxis - metricTime) * yScale + topGraphPadding);
graphPoints.add(new Point(x1, y1));
}
// Sort the points
Collections.sort(graphPoints, new Comparator<Point>() {
@Override
public int compare(Point o1, Point o2) {
if(o1.getX() > o2.getX()) {
return 1;
} else if (o1.getX() < o2.getX()) {
return -1;
}
return 0;
}
});
// Draw the selected type of graph. If there's only one data point,
// draw that single point.
g2.setStroke(NARROW_STROKE);
g2.setColor(lineColor);
if(doLineGraph && graphPoints.size() > 1) {
for (int i = 0; i < graphPoints.size() - 1; i++) {
int x1 = graphPoints.get(i).x;
int y1 = graphPoints.get(i).y;
int x2 = graphPoints.get(i + 1).x;
int y2 = graphPoints.get(i + 1).y;
g2.drawLine(x1, y1, x2, y2);
}
} else {
for (int i = 0; i < graphPoints.size(); i++) {
int x = graphPoints.get(i).x - pointWidth / 2;
int y = graphPoints.get(i).y - pointWidth / 2;
int ovalW = pointWidth;
int ovalH = pointWidth;
g2.fillOval(x, y, ovalW, ovalH);
}
}
// Draw the trend line.
// Don't draw anything if we don't have at least two data points.
if(showTrendLine && (trendLine != null) && (timingResults.size() > 1)) {
double x0value = minValueOnXAxis;
double y0value = trendLine.getExpectedValueAt(x0value);
if (y0value < minValueOnYAxis) {
try {
y0value = minValueOnYAxis;
x0value = trendLine.getXGivenY(y0value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
} else if (y0value > maxValueOnYAxis) {
try {
y0value = maxValueOnYAxis;
x0value = trendLine.getXGivenY(y0value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
}
int x0 = (int) ((x0value - minValueOnXAxis) * xScale) + leftGraphPadding;
int y0 = (int) ((maxValueOnYAxis - y0value) * yScale + topGraphPadding);
double x1value = maxValueOnXAxis;
double y1value = trendLine.getExpectedValueAt(maxValueOnXAxis);
if (y1value < minValueOnYAxis) {
try {
y1value = minValueOnYAxis;
x1value = trendLine.getXGivenY(y1value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
} else if (y1value > maxValueOnYAxis) {
try {
y1value = maxValueOnYAxis;
x1value = trendLine.getXGivenY(y1value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
}
int x1 = (int) ((x1value - minValueOnXAxis) * xScale) + leftGraphPadding;
int y1 = (int) ((maxValueOnYAxis - y1value) * yScale + topGraphPadding);
g2.setStroke(GRAPH_STROKE);
g2.setColor(trendLineColor);
g2.drawLine(x0, y0, x1, y1);
}
// The graph lines may have extended up past the bounds of the graph. Overwrite that
// area with the original background color.
g2.setColor(this.getBackground());
g2.fillRect(leftGraphPadding, 0, graphWidth, topGraphPadding);
// Write the scale. Do this after we erase the top block of the graph.
g2.setColor(Color.BLACK);
String scaleStr = Bundle.TimingMetricGraphPanel_paintComponent_displayingTime() + yUnitString;
String titleStr = metricName + " - " + scaleStr;
g2.drawString(titleStr, positionForMetricNameLabel, padding);
}
/**
* Class to generate a linear trend line from timing metric data.
*
* Formula for the linear trend line:
* (x,y) = (timestamp, metric time)
* n = total number of metrics
*
* slope = ( n * Σ(xy) - Σx * Σy ) / ( n * Σ(x^2) - (Σx)^2 )
*
* y intercept = ( Σy - (slope) * Σx ) / n
*/
private class TrendLine {
double slope;
double yInt;
TrendLine(List<DatabaseTimingResult> timingResults) throws HealthMonitorException {
if((timingResults == null) || timingResults.isEmpty()) {
throw new HealthMonitorException("Can not generate trend line for empty/null data set");
}
// Calculate intermediate values
int n = timingResults.size();
double sumX = 0;
double sumY = 0;
double sumXY = 0;
double sumXsquared = 0;
for(int i = 0;i < n;i++) {
double x = timingResults.get(i).getTimestamp();
double y = timingResults.get(i).getAverage();
sumX += x;
sumY += y;
sumXY += x * y;
sumXsquared += x * x;
}
// Calculate slope
// With only one measurement, the denominator will end being zero in the formula.
// Use a horizontal line in this case (or any case where the denominator is zero)
double denominator = n * sumXsquared - sumX * sumX;
if (denominator != 0) {
slope = (n * sumXY - sumX * sumY) / denominator;
} else {
slope = 0;
}
// Calculate y intercept
yInt = (sumY - slope * sumX) / n;
}
/**
* Get the expected y value for a given x
* @param x x coordinate of the point on the trend line
* @return expected y coordinate of this point on the trend line
*/
double getExpectedValueAt(double x) {
return (slope * x + yInt);
}
/**
* Get the x-coordinate for a given Y-coordinate.
* Should only be necessary when the trend line does not fit on the graph
* @param y the y coordinate
* @return expected x coordinate for the given y
* @throws HealthMonitorException
*/
double getXGivenY(double y) throws HealthMonitorException {
if (slope != 0.0) {
return ((y - yInt) / slope);
} else {
throw new HealthMonitorException("Attempted division by zero in trend line calculation");
}
}
}
}
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java
+71 -48
View File
@@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -37,6 +38,9 @@ import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestPipeline.PipelineModule;
import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason;
import org.sleuthkit.autopsy.ingest.IngestTasksScheduler.IngestJobTasksSnapshot;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.IngestJobInfo;
@@ -51,7 +55,7 @@ import org.sleuthkit.autopsy.modules.interestingitems.FilesSet;
* Encapsulates a data source and the ingest module pipelines used to process
* it.
*/
final class DataSourceIngestJob {
public final class DataSourceIngestJob {
private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
@@ -1079,71 +1083,90 @@ final class DataSourceIngestJob {
* @return An ingest job statistics object.
*/
Snapshot getSnapshot(boolean getIngestTasksSnapshot) {
return new Snapshot(getIngestTasksSnapshot);
/**
* Determine whether file ingest is running at the time of this snapshot
* and determine the earliest file ingest level pipeline start time, if
* file ingest was started at all.
*/
boolean fileIngestRunning = false;
Date fileIngestStartTime = null;
for (FileIngestPipeline pipeline : this.fileIngestPipelines) {
if (pipeline.isRunning()) {
fileIngestRunning = true;
}
Date pipelineStartTime = pipeline.getStartTime();
if (null != pipelineStartTime && (null == fileIngestStartTime || pipelineStartTime.before(fileIngestStartTime))) {
fileIngestStartTime = pipelineStartTime;
}
}
long processedFilesCount = 0;
long estimatedFilesToProcessCount = 0;
long snapShotTime = new Date().getTime();
IngestJobTasksSnapshot tasksSnapshot = null;
if (getIngestTasksSnapshot) {
synchronized (fileIngestProgressLock) {
processedFilesCount = this.processedFiles;
estimatedFilesToProcessCount = this.estimatedFilesToProcess;
snapShotTime = new Date().getTime();
}
tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(id);
}
return new Snapshot(this.dataSource.getName(), id, createTime,
getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime,
cancelled, cancellationReason, cancelledDataSourceIngestModules,
processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot);
}
/**
* Stores basic diagnostic statistics for a data source ingest job.
*/
final class Snapshot {
public static final class Snapshot implements Serializable {
private static final long serialVersionUID = 1L;
private final String dataSource;
private final long jobId;
private final long jobStartTime;
private final long snapShotTime;
private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
private boolean fileIngestRunning;
private Date fileIngestStartTime;
transient private final PipelineModule dataSourceLevelIngestModule;
private final boolean fileIngestRunning;
private final Date fileIngestStartTime;
private final long processedFiles;
private final long estimatedFilesToProcess;
private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot;
private final boolean jobCancelled;
private final IngestJob.CancellationReason jobCancellationReason;
private final List<String> cancelledDataSourceModules;
private final IngestJobTasksSnapshot tasksSnapshot;
transient private final boolean jobCancelled;
transient private final CancellationReason jobCancellationReason;
transient private final List<String> cancelledDataSourceModules;
/**
* Constructs an object to store basic diagnostic statistics for a data
* source ingest job.
*/
Snapshot(boolean getIngestTasksSnapshot) {
this.dataSource = DataSourceIngestJob.this.dataSource.getName();
this.jobId = DataSourceIngestJob.this.id;
this.jobStartTime = DataSourceIngestJob.this.createTime;
this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule();
Snapshot(String dataSourceName, long jobId, long jobStartTime, PipelineModule dataSourceIngestModule,
boolean fileIngestRunning, Date fileIngestStartTime,
boolean jobCancelled, CancellationReason cancellationReason, List<String> cancelledModules,
long processedFiles, long estimatedFilesToProcess,
long snapshotTime, IngestJobTasksSnapshot tasksSnapshot) {
this.dataSource = dataSourceName;
this.jobId = jobId;
this.jobStartTime = jobStartTime;
this.dataSourceLevelIngestModule = dataSourceIngestModule;
/**
* Determine whether file ingest is running at the time of this
* snapshot and determine the earliest file ingest level pipeline
* start time, if file ingest was started at all.
*/
for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) {
if (pipeline.isRunning()) {
this.fileIngestRunning = true;
}
Date pipelineStartTime = pipeline.getStartTime();
if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(this.fileIngestStartTime))) {
this.fileIngestStartTime = pipelineStartTime;
}
}
this.jobCancelled = cancelled;
this.fileIngestRunning = fileIngestRunning;
this.fileIngestStartTime = fileIngestStartTime;
this.jobCancelled = jobCancelled;
this.jobCancellationReason = cancellationReason;
this.cancelledDataSourceModules = new ArrayList<>(DataSourceIngestJob.this.cancelledDataSourceIngestModules);
this.cancelledDataSourceModules = cancelledModules;
if (getIngestTasksSnapshot) {
synchronized (DataSourceIngestJob.this.fileIngestProgressLock) {
this.processedFiles = DataSourceIngestJob.this.processedFiles;
this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess;
this.snapShotTime = new Date().getTime();
}
this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId);
} else {
this.processedFiles = 0;
this.estimatedFilesToProcess = 0;
this.snapShotTime = new Date().getTime();
this.tasksSnapshot = null;
}
this.processedFiles = processedFiles;
this.estimatedFilesToProcess = estimatedFilesToProcess;
this.snapShotTime = snapshotTime;
this.tasksSnapshot = tasksSnapshot;
}
/**
@@ -1190,11 +1213,11 @@ final class DataSourceIngestJob {
return this.dataSourceLevelIngestModule;
}
boolean fileIngestIsRunning() {
boolean getFileIngestIsRunning() {
return this.fileIngestRunning;
}
Date fileIngestStartTime() {
Date getFileIngestStartTime() {
return this.fileIngestStartTime;
}
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
+2 -2
View File
@@ -355,10 +355,10 @@ public final class IngestJob {
dataSourceModule = new DataSourceIngestModuleHandle(dataSourceJobs.get(snapshot.getJobId()), module);
}
}
if (snapshot.fileIngestIsRunning()) {
if (snapshot.getFileIngestIsRunning()) {
fileIngestRunning = true;
}
Date childFileIngestStartTime = snapshot.fileIngestStartTime();
Date childFileIngestStartTime = snapshot.getFileIngestStartTime();
if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
fileIngestStartTime = childFileIngestStartTime;
}
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
+11 -5
View File
@@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.awt.EventQueue;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -107,7 +108,7 @@ import org.sleuthkit.datamodel.Content;
* job progress, and ingest module run times.
*/
@ThreadSafe
public class IngestManager {
public class IngestManager implements IngestProgressSnapshotProvider {
private final static Logger logger = Logger.getLogger(IngestManager.class.getName());
private final static String INGEST_JOB_EVENT_CHANNEL_NAME = "%s-Ingest-Job-Events"; //NON-NLS
@@ -756,7 +757,8 @@ public class IngestManager {
*
* @return Map of module name to run time (in milliseconds)
*/
Map<String, Long> getModuleRunTimes() {
@Override
public Map<String, Long> getModuleRunTimes() {
synchronized (ingestModuleRunTimes) {
Map<String, Long> times = new HashMap<>(ingestModuleRunTimes);
return times;
@@ -769,7 +771,8 @@ public class IngestManager {
*
* @return A collection of ingest manager ingest task snapshots.
*/
List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
@Override
public List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
return new ArrayList<>(ingestThreadActivitySnapshots.values());
}
@@ -778,7 +781,8 @@ public class IngestManager {
*
* @return A list of ingest job state snapshots.
*/
List<DataSourceIngestJob.Snapshot> getIngestJobSnapshots() {
@Override
public List<DataSourceIngestJob.Snapshot> getIngestJobSnapshots() {
List<DataSourceIngestJob.Snapshot> snapShots = new ArrayList<>();
synchronized (ingestJobsById) {
ingestJobsById.values().forEach((job) -> {
@@ -916,7 +920,9 @@ public class IngestManager {
* running in an ingest thread.
*/
@Immutable
static final class IngestThreadActivitySnapshot {
public static final class IngestThreadActivitySnapshot implements Serializable {
private static final long serialVersionUID = 1L;
private final long threadId;
private final Date startTime;
Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java
+17 -2
View File
@@ -50,12 +50,14 @@ public final class IngestProgressSnapshotDialog extends JDialog {
/**
* Constructs an instance of the dialog with its own frame. Could be modal.
* Uses the given provider as the source of data for the dialog.
*
* @param owner - the owner of this dialog. If this dialog should be
* modal, the owner gets set to non modal.
* @param shouldBeModal - true if this should be modal, false otherwise.
* @param provider - the provider to use as the source of data.
*/
public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) {
public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal, IngestProgressSnapshotProvider provider) {
super((Window) owner, TITLE, ModalityType.MODELESS);
if (shouldBeModal && owner instanceof JDialog) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal.
final JDialog pseudoOwner = (JDialog) owner;
@@ -82,7 +84,7 @@ public final class IngestProgressSnapshotDialog extends JDialog {
this.getRootPane().registerKeyboardAction(e -> {
this.dispose();
}, KeyStroke.getKeyStroke(KeyEvent.VK_ESCAPE, 0), JComponent.WHEN_IN_FOCUSED_WINDOW);
add(new IngestProgressSnapshotPanel(this));
add(new IngestProgressSnapshotPanel(this, provider));
pack();
setResizable(false);
if (shouldBeModal) { // if called from a modal dialog, become modal, otherwise don't.
@@ -90,4 +92,17 @@ public final class IngestProgressSnapshotDialog extends JDialog {
}
setVisible(true);
}
/**
* Constructs an instance of the dialog with its own frame. Could be modal.
* Uses the internal IngestManager instance as the source of data for the
* dialog
*
* @param owner - the owner of this dialog. If this dialog should be
* modal, the owner gets set to non modal.
* @param shouldBeModal - true if this should be modal, false otherwise.
*/
public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) {
this(owner, shouldBeModal, IngestManager.getInstance());
}
}
Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java
+7 -5
View File
@@ -33,15 +33,17 @@ import org.openide.util.NbBundle;
/**
* A panel that displays ingest task progress snapshots.
*/
public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
class IngestProgressSnapshotPanel extends javax.swing.JPanel {
private final JDialog parent;
private final IngestProgressSnapshotProvider snapshotProvider;
private final IngestThreadActivitySnapshotsTableModel threadActivityTableModel;
private final IngestJobTableModel jobTableModel;
private final ModuleTableModel moduleTableModel;
IngestProgressSnapshotPanel(JDialog parent) {
IngestProgressSnapshotPanel(JDialog parent, IngestProgressSnapshotProvider snapshotProvider) {
this.parent = parent;
this.snapshotProvider = snapshotProvider;
threadActivityTableModel = new IngestThreadActivitySnapshotsTableModel();
jobTableModel = new IngestJobTableModel();
moduleTableModel = new ModuleTableModel();
@@ -105,7 +107,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
}
private void refresh() {
snapshots = IngestManager.getInstance().getIngestThreadActivitySnapshots();
snapshots = snapshotProvider.getIngestThreadActivitySnapshots();
fireTableDataChanged();
}
@@ -187,7 +189,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
}
private void refresh() {
jobSnapshots = IngestManager.getInstance().getIngestJobSnapshots();
jobSnapshots = snapshotProvider.getIngestJobSnapshots();
fireTableDataChanged();
}
@@ -299,7 +301,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
}
private void refresh() {
Map<String, Long> moduleStatMap = IngestManager.getInstance().getModuleRunTimes();
Map<String, Long> moduleStatMap = snapshotProvider.getModuleRunTimes();
moduleStats.clear();
totalTime = 0;
for (String k : moduleStatMap.keySet()) {
Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotProvider.java
+49
View File
@@ -0,0 +1,49 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.List;
import java.util.Map;
/**
* Interface that provides a snapshot of ingest progress.
*/
public interface IngestProgressSnapshotProvider {
/**
* Get a snapshot of the state of ingest threads.
*
* @return A list of IngestThreadActivitySnapshot
*/
List<IngestManager.IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots();
/**
* Get a snapshot of the state of ingest jobs.
*
* @return A list of ingest job snapshots.
*/
List<DataSourceIngestJob.Snapshot> getIngestJobSnapshots();
/**
* Gets the cumulative run times for the ingest module.
*
* @return Map of module name to run time (in milliseconds)
*/
Map<String, Long> getModuleRunTimes();
}
Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java
+14 -8
View File
@@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -555,7 +556,11 @@ final class IngestTasksScheduler {
* @return
*/
synchronized IngestJobTasksSnapshot getTasksSnapshotForJob(long jobId) {
return new IngestJobTasksSnapshot(jobId);
return new IngestJobTasksSnapshot(jobId, this.dataSourceIngestThreadQueue.countQueuedTasksForJob(jobId),
countTasksForJob(this.rootFileTaskQueue, jobId),
countTasksForJob(this.pendingFileTaskQueue, jobId),
this.fileIngestThreadsQueue.countQueuedTasksForJob(jobId),
this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + this.fileIngestThreadsQueue.countRunningTasksForJob(jobId));
}
/**
@@ -825,8 +830,9 @@ final class IngestTasksScheduler {
/**
* A snapshot of ingest tasks data for an ingest job.
*/
class IngestJobTasksSnapshot {
static final class IngestJobTasksSnapshot implements Serializable {
private static final long serialVersionUID = 1L;
private final long jobId;
private final long dsQueueSize;
private final long rootQueueSize;
@@ -839,13 +845,13 @@ final class IngestTasksScheduler {
*
* @param jobId The identifier associated with the job.
*/
IngestJobTasksSnapshot(long jobId) {
IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, long runningListSize) {
this.jobId = jobId;
this.dsQueueSize = IngestTasksScheduler.this.dataSourceIngestThreadQueue.countQueuedTasksForJob(jobId);
this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootFileTaskQueue, jobId);
this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingFileTaskQueue, jobId);
this.fileQueueSize = IngestTasksScheduler.this.fileIngestThreadsQueue.countQueuedTasksForJob(jobId);;
this.runningListSize = IngestTasksScheduler.this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + IngestTasksScheduler.this.fileIngestThreadsQueue.countRunningTasksForJob(jobId);
this.dsQueueSize = dsQueueSize;
this.rootQueueSize = rootQueueSize;
this.dirQueueSize = dirQueueSize;
this.fileQueueSize = fileQueueSize;
this.runningListSize = runningListSize;
}
/**
Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties
+2 -2
View File
@@ -9,8 +9,8 @@ OpenIDE-Module-Name=Embedded File Extraction
OpenIDE-Module-Short-Description=Embedded File Extraction Ingest Module
EmbeddedFileExtractorIngestModule.SevenZipContentReadStream.seek.exception.invalidOrigin=Invalid seek origin\: {0}
EmbeddedFileExtractorIngestModule.SevenZipContentReadStream.read.exception.errReadStream=Error reading content stream.
EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel=File-level Encryption
EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull=Full Encryption
EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel=Content-only Encryption (Archive File)
EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull=Full Encryption (Archive File)
EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errInitModule.details=Error initializing output dir\: {0}\: {1}
EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg=Possible ZIP bomb detected in archive\: {0}, item\: {1}
EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails=Compression ratio is {0}, skipping item in {1}.
Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java
+24 -11
View File
@@ -65,6 +65,9 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
private static final int FILE_SIZE_MODULUS = 512;
private static final String DATABASE_FILE_EXTENSION = "db";
private static final int MINIMUM_DATABASE_FILE_SIZE = 65536; //64 KB
private static final String MIME_TYPE_OOXML_PROTECTED = "application/x-ooxml-protected";
private static final String MIME_TYPE_MSWORD = "application/msword";
private static final String MIME_TYPE_MSEXCEL = "application/vnd.ms-excel";
@@ -141,8 +144,7 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
}
} else {
if (isFilePasswordProtected(file)) {
return flagFile(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED,
Bundle.EncryptionDetectionFileIngestModule_artifactComment_password());
return flagFile(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED, Bundle.EncryptionDetectionFileIngestModule_artifactComment_password());
}
}
}
@@ -182,7 +184,6 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
private IngestModule.ProcessResult flagFile(AbstractFile file, BlackboardArtifact.ARTIFACT_TYPE artifactType, String comment) {
try {
BlackboardArtifact artifact = file.newArtifact(artifactType);
artifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT,
EncryptionDetectionModuleFactory.getModuleName(), comment));
@@ -362,18 +363,30 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
/*
* Qualify the size.
*/
boolean fileSizeQualified = false;
String fileExtension = file.getNameExtension();
long contentSize = file.getSize();
if (contentSize >= minimumFileSize) {
// Database files qualify at 64 KB minimum for SQLCipher detection.
if (fileExtension.equalsIgnoreCase(DATABASE_FILE_EXTENSION)) {
if (contentSize >= MINIMUM_DATABASE_FILE_SIZE) {
fileSizeQualified = true;
}
} else if (contentSize >= minimumFileSize) {
if (!fileSizeMultipleEnforced || (contentSize % FILE_SIZE_MODULUS) == 0) {
/*
* Qualify the entropy.
*/
calculatedEntropy = EncryptionDetectionTools.calculateEntropy(file);
if (calculatedEntropy >= minimumEntropy) {
possiblyEncrypted = true;
}
fileSizeQualified = true;
}
}
if (fileSizeQualified) {
/*
* Qualify the entropy.
*/
calculatedEntropy = EncryptionDetectionTools.calculateEntropy(file);
if (calculatedEntropy >= minimumEntropy) {
possiblyEncrypted = true;
}
}
return possiblyEncrypted;
}
}
Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupModuleFactory.java
-2
View File
@@ -18,8 +18,6 @@
*/
package org.sleuthkit.autopsy.modules.hashdatabase;
import java.util.ArrayList;
import java.util.List;
import org.openide.util.NbBundle;
import org.openide.util.lookup.ServiceProvider;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java
Regular → Executable
View File
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithHashAndFileType.java
+465
View File
@@ -0,0 +1,465 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilessearch;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import junit.framework.Test;
import org.netbeans.junit.NbModuleSuite;
import org.netbeans.junit.NbTestCase;
import org.openide.util.Exceptions;
import org.python.icu.impl.Assert;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm;
import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata;
import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder;
import org.sleuthkit.autopsy.commonfilesearch.SingleDataSource;
import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.*;
import org.sleuthkit.autopsy.ingest.IngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestJobSettings.IngestType;
import org.sleuthkit.autopsy.ingest.IngestModuleTemplate;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory;
import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory;
import org.sleuthkit.autopsy.testutils.IngestUtils;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Add set 1, set 2, set 3, and set 4 to case and ingest with hash algorithm.
*/
public class IngestedWithHashAndFileType extends NbTestCase {
public static Test suite() {
NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(IngestedWithHashAndFileType.class).
clusters(".*").
enableModules(".*");
return conf.suite();
}
private final IntraCaseUtils utils;
public IngestedWithHashAndFileType(String name) {
super(name);
this.utils = new IntraCaseUtils(this, "IngestedWithHashAndFileTypeTests");
}
@Override
public void setUp() {
this.utils.setUp();
IngestModuleTemplate hashLookupTemplate = IngestUtils.getIngestModuleTemplate(new HashLookupModuleFactory());
IngestModuleTemplate mimeTypeLookupTemplate = IngestUtils.getIngestModuleTemplate(new FileTypeIdModuleFactory());
ArrayList<IngestModuleTemplate> templates = new ArrayList<>();
templates.add(hashLookupTemplate);
templates.add(mimeTypeLookupTemplate);
IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestedWithHashAndFileType.class.getCanonicalName(), IngestType.FILES_ONLY, templates);
try {
IngestUtils.runIngestJob(Case.getCurrentCaseThrows().getDataSources(), ingestJobSettings);
} catch (NoCurrentCaseException | TskCoreException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
@Override
public void tearDown() {
this.utils.tearDown();
}
/**
* Find all matches & all file types. Confirm file.jpg is found on all three
* and file.docx is found on two.
*/
public void testOneA() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false);
CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 2));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find all matches & only image types. Confirm file.jpg is found on all
* three.
*/
public void testOneB() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, true, false);
CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 2));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find all matches & only image types. Confirm file.jpg is found on all
* three.
*/
public void testOneC() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, true);
CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find matches on set 1 & all file types. Confirm same results.
*
*/
public void testTwoA() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long first = getDataSourceIdByName(SET1, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(first, dataSources, false, false);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 2));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find matches on set 1 & only media types. Confirm same results.
*
*/
public void testTwoB() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long first = getDataSourceIdByName(SET1, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(first, dataSources, true, false);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 2));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find matches on set 1 & all file types. Confirm same results.
*
*/
public void testTwoC() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long first = getDataSourceIdByName(SET1, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(first, dataSources, false, true);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find matches on set 2 & all file types: Confirm file.jpg.
*
*/
public void testThree() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long second = getDataSourceIdByName(SET2, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(second, dataSources, false, false);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 2));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find matches on set 4 & all file types: Confirm nothing is found.
*/
public void testFour() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long last = getDataSourceIdByName(SET4, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(last, dataSources, false, false);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find matches on set 3 & all file types: Confirm file.jpg and file.docx.
*/
public void testFive() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long third = getDataSourceIdByName(SET3, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(third, dataSources, false, false);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = getFiles(objectIdToDataSource.keySet());
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET1, 2));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET2, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, IMG, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET1, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET3, 1));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, DOC, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, PDF, SET4, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET1, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET2, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET3, 0));
assertTrue(verifyFileExistanceAndCount(files, objectIdToDataSource, EMPTY, SET4, 0));
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
}
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithNoFileTypes.java
+140
View File
@@ -0,0 +1,140 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilessearch;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static junit.framework.Assert.assertTrue;
import junit.framework.Test;
import org.netbeans.junit.NbModuleSuite;
import org.netbeans.junit.NbTestCase;
import org.openide.util.Exceptions;
import org.python.icu.impl.Assert;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm;
import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata;
import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder;
import org.sleuthkit.autopsy.commonfilesearch.SingleDataSource;
import org.sleuthkit.autopsy.ingest.IngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestModuleTemplate;
import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory;
import org.sleuthkit.autopsy.testutils.IngestUtils;
import static org.sleuthkit.autopsy.testutils.IngestUtils.getIngestModuleTemplate;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Ingested w/o mime type info added to DB.
*
* Setup:
*
* Add images set 1, set 2, set 3, and set 4 to case. Do not run mime type
* module.
*/
public class IngestedWithNoFileTypes extends NbTestCase {
public static Test suite() {
NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(IngestedWithNoFileTypes.class).
clusters(".*").
enableModules(".*");
return conf.suite();
}
private final IntraCaseUtils utils;
public IngestedWithNoFileTypes(String name) {
super(name);
this.utils = new IntraCaseUtils(this, "IngestedWithNoFileTypes");
}
@Override
public void setUp() {
this.utils.setUp();
IngestModuleTemplate hashLookupTemplate = getIngestModuleTemplate(new HashLookupModuleFactory());
ArrayList<IngestModuleTemplate> templates = new ArrayList<>();
templates.add(hashLookupTemplate);
IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestedWithHashAndFileType.class.getCanonicalName(), IngestJobSettings.IngestType.FILES_ONLY, templates);
try {
IngestUtils.runIngestJob(Case.getCurrentCaseThrows().getDataSources(), ingestJobSettings);
} catch (NoCurrentCaseException | TskCoreException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
@Override
public void tearDown(){
this.utils.tearDown();
}
/**
* Search using all data sources and filtering for media types. We should
* find nothing and no errors should arise.
*/
public void testOne() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, true, false);
CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet());
assertTrue(files.isEmpty());
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
}
}
/**
* Search using single data source and filtering for doc types. Observe that
* nothing is found and that nothing blows up.
*/
public void testTwo() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long third = IntraCaseUtils.getDataSourceIdByName(IntraCaseUtils.SET3, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(third, dataSources, true, false);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
Map<Long, String> objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata);
List<AbstractFile> files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet());
assertTrue(files.isEmpty());
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
}
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java
+220
View File
@@ -0,0 +1,220 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilessearch;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.netbeans.junit.NbTestCase;
import org.openide.util.Exceptions;
import org.python.icu.impl.Assert;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.ImageDSProcessor;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata;
import org.sleuthkit.autopsy.commonfilesearch.DataSourceLoader;
import org.sleuthkit.autopsy.commonfilesearch.FileInstanceMetadata;
import org.sleuthkit.autopsy.commonfilesearch.Md5Metadata;
import org.sleuthkit.autopsy.testutils.CaseUtils;
import org.sleuthkit.autopsy.testutils.IngestUtils;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
/**
*
* Provides setup and utility for testing presence of files in different data
* sets discoverable by Common Files Features.
*
* Data set definitions:
*
* set 1
* + file1
* - IMG_6175.jpg
* + file2
* - IMG_6175.jpg
* + file3
* - BasicStyleGuide.doc
*
* set 2
* - adsf.pdf
* - IMG_6175.jpg
*
* set 3
* - BasicStyleGuide.doc
* - IMG_6175.jpg
*
* set 4
* - file.dat (empty file)
*/
class IntraCaseUtils {
private static final String CASE_NAME = "IntraCaseCommonFilesSearchTest";
static final Path CASE_DIRECTORY_PATH = Paths.get(System.getProperty("java.io.tmpdir"), CASE_NAME);
private final Path imagePath1;
private final Path imagePath2;
private final Path imagePath3;
private final Path imagePath4;
static final String IMG = "IMG_6175.jpg";
static final String DOC = "BasicStyleGuide.doc";
static final String PDF = "adsf.pdf"; //not a typo - it appears this way in the test image
static final String EMPTY = "file.dat";
static final String SET1 = "commonfiles_image1_v1.vhd";
static final String SET2 = "commonfiles_image2_v1.vhd";
static final String SET3 = "commonfiles_image3_v1.vhd";
static final String SET4 = "commonfiles_image4_v1.vhd";
private final DataSourceLoader dataSourceLoader;
private final String caseName;
IntraCaseUtils(NbTestCase nbTestCase, String caseName){
imagePath1 = Paths.get(nbTestCase.getDataDir().toString(), "commonfiles_image1_v1.vhd");
imagePath2 = Paths.get(nbTestCase.getDataDir().toString(), "commonfiles_image2_v1.vhd");
imagePath3 = Paths.get(nbTestCase.getDataDir().toString(), "commonfiles_image3_v1.vhd");
imagePath4 = Paths.get(nbTestCase.getDataDir().toString(), "commonfiles_image4_v1.vhd");
this.dataSourceLoader = new DataSourceLoader();
this.caseName = caseName;
}
void setUp(){
CaseUtils.createAsCurrentCase(this.caseName);
final ImageDSProcessor imageDSProcessor = new ImageDSProcessor();
IngestUtils.addDataSource(imageDSProcessor, imagePath1);
IngestUtils.addDataSource(imageDSProcessor, imagePath2);
IngestUtils.addDataSource(imageDSProcessor, imagePath3);
IngestUtils.addDataSource(imageDSProcessor, imagePath4);
}
Map<Long, String> getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException{
return this.dataSourceLoader.getDataSourceMap();
}
void tearDown(){
CaseUtils.closeCurrentCase(false);
try {
CaseUtils.deleteCaseDir(CASE_DIRECTORY_PATH.toFile());
} catch (IOException ex) {
Exceptions.printStackTrace(ex);
//does not represent a failure in the common files search feature
}
}
/**
* Verify that the given file appears a precise number times in the given
* data source.
*
* @param files search domain
* @param objectIdToDataSource mapping of file ids to data source names
* @param name name of file to search for
* @param dataSource name of data source where file should appear
* @param count number of appearances of the given file
* @return true if a file with the given name exists the specified number
* of times in the given data source
*/
static boolean verifyFileExistanceAndCount(List<AbstractFile> files, Map<Long, String> objectIdToDataSource, String name, String dataSource, int count) {
int tally = 0;
for (AbstractFile file : files) {
Long objectId = file.getId();
String fileName = file.getName();
String dataSourceName = objectIdToDataSource.get(objectId);
if (fileName.equals(name) && dataSourceName.equals(dataSource)) {
tally++;
}
}
return tally == count;
}
/**
* Convenience method which verifies that a file exists within a given data
* source exactly once.
*
* @param files search domain
* @param objectIdToDataSource mapping of file ids to data source names
* @param name name of file to search for
* @param dataSource name of data source where file should appear
* @return true if a file with the given name exists once in the given data
* source
*/
static boolean verifySingularFileExistance(List<AbstractFile> files, Map<Long, String> objectIdToDataSource, String name, String dataSource) {
return verifyFileExistanceAndCount(files, objectIdToDataSource, name, dataSource, 1);
}
static Map<Long, String> mapFileInstancesToDataSources(CommonFilesMetadata metadata) {
Map<Long, String> instanceIdToDataSource = new HashMap<>();
for (Map.Entry<String, Md5Metadata> entry : metadata.getMetadata().entrySet()) {
for (FileInstanceMetadata md : entry.getValue().getMetadata()) {
instanceIdToDataSource.put(md.getObjectId(), md.getDataSourceName());
}
}
return instanceIdToDataSource;
}
static List<AbstractFile> getFiles(Set<Long> objectIds) {
List<AbstractFile> files = new ArrayList<>(objectIds.size());
for (Long id : objectIds) {
try {
AbstractFile file = Case.getCurrentCaseThrows().getSleuthkitCase().getAbstractFileById(id);
files.add(file);
} catch (NoCurrentCaseException | TskCoreException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
return files;
}
static Long getDataSourceIdByName(String name, Map<Long, String> dataSources){
if(dataSources.containsValue(name)){
for(Map.Entry<Long, String> dataSource : dataSources.entrySet()){
if(dataSource.getValue().equals(name)){
return dataSource.getKey();
}
}
} else {
throw new IndexOutOfBoundsException(String.format("Name should be one of: {0}", String.join(",", dataSources.values())));
}
return null;
}
}
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/UningestedCases.java
+114
View File
@@ -0,0 +1,114 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilessearch;
import java.sql.SQLException;
import java.util.Map;
import static junit.framework.Assert.assertEquals;
import junit.framework.Test;
import org.netbeans.junit.NbModuleSuite;
import org.netbeans.junit.NbTestCase;
import org.openide.util.Exceptions;
import org.python.icu.impl.Assert;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm;
import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata;
import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder;
import org.sleuthkit.autopsy.commonfilesearch.SingleDataSource;
import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET1;
import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.getDataSourceIdByName;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Test that cases which are created but have not run any ingest modules turn up
* no results.
*
* Setup:
*
* Add images set 1, set 2, set 3, and set 4 to case. Do not ingest.
*
*/
public class UningestedCases extends NbTestCase {
public static Test suite() {
NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(UningestedCases.class).
clusters(".*").
enableModules(".*");
return conf.suite();
}
private final IntraCaseUtils utils;
public UningestedCases(String name) {
super(name);
this.utils = new IntraCaseUtils(this, "UningestedCasesTests");
}
@Override
public void setUp(){
this.utils.setUp();
}
@Override
public void tearDown(){
this.utils.tearDown();
}
/**
* Find all matches & all file types. Confirm no matches are found (since
* there are no hashes to match).
*/
public void testOne() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false);
CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles();
int resultCount = metadata.size();
assertEquals(resultCount, 0);
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
/**
* Find all matches on image #1 & all file types. Confirm no matches.
*/
public void testTwo() {
try {
Map<Long, String> dataSources = this.utils.getDataSourceMap();
Long first = getDataSourceIdByName(SET1, dataSources);
CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(first, dataSources, false, false);
CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles();
int resultCount = metadata.size();
assertEquals(resultCount, 0);
} catch (NoCurrentCaseException | TskCoreException | SQLException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
}
Core/test/qa-functional/src/org/sleuthkit/autopsy/ingest/IngestFileFiltersTest.java
Regular → Executable
View File
Core/test/qa-functional/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionTest.java
+68 -4
View File
@@ -52,10 +52,12 @@ public class EncryptionDetectionTest extends NbTestCase {
private static final String BITLOCKER_DETECTION_CASE_NAME = "testBitlockerEncryption";
private static final String PASSWORD_DETECTION_CASE_NAME = "PasswordDetectionTest";
private static final String VERACRYPT_DETECTION_CASE_NAME = "VeraCryptDetectionTest";
private static final String SQLCIPHER_DETECTION_CASE_NAME = "SQLCipherDetectionTest";
private final Path BITLOCKER_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "encryption_detection_bitlocker_test.vhd");
private final Path PASSWORD_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "password_detection_test.img");
private final Path VERACRYPT_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "veracrypt_detection_test.vhd");
private final Path SQLCIPHER_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "encryption_detection_sqlcipher_test.vhd");
private boolean testSucceeded;
@@ -79,7 +81,7 @@ public class EncryptionDetectionTest extends NbTestCase {
public void tearDown() {
CaseUtils.closeCurrentCase(testSucceeded);
}
/**
* Test the Encryption Detection module's volume encryption detection.
*/
@@ -88,9 +90,9 @@ public class EncryptionDetectionTest extends NbTestCase {
Case openCase = CaseUtils.createAsCurrentCase(BITLOCKER_DETECTION_CASE_NAME);
ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
IngestUtils.addDataSource(dataSourceProcessor, BITLOCKER_DETECTION_IMAGE_PATH);
/*
* Create ingest job settings.
* Create ingest job settings and run ingest job.
*/
IngestModuleFactory ingestModuleFactory = new EncryptionDetectionModuleFactory();
IngestModuleIngestJobSettings settings = ingestModuleFactory.getDefaultIngestJobSettings();
@@ -164,7 +166,6 @@ public class EncryptionDetectionTest extends NbTestCase {
/*
* Create ingest job settings.
*/
ArrayList<IngestModuleTemplate> templates = new ArrayList<>();
templates.add(IngestUtils.getIngestModuleTemplate(new EncryptionDetectionModuleFactory()));
IngestJobSettings ingestJobSettings = new IngestJobSettings(PASSWORD_DETECTION_CASE_NAME, IngestType.FILES_ONLY, templates);
@@ -295,4 +296,67 @@ public class EncryptionDetectionTest extends NbTestCase {
testSucceeded = true;
}
/**
* Test the Encryption Detection module's SQLCipher encryption detection.
*/
public void testSqlCipherEncryption() {
try {
Case openCase = CaseUtils.createAsCurrentCase(SQLCIPHER_DETECTION_CASE_NAME);
ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
IngestUtils.addDataSource(dataSourceProcessor, SQLCIPHER_DETECTION_IMAGE_PATH);
/*
* Create ingest job settings.
*/
ArrayList<IngestModuleTemplate> templates = new ArrayList<>();
templates.add(IngestUtils.getIngestModuleTemplate(new EncryptionDetectionModuleFactory()));
IngestJobSettings ingestJobSettings = new IngestJobSettings(SQLCIPHER_DETECTION_CASE_NAME, IngestType.FILES_ONLY, templates);
IngestUtils.runIngestJob(openCase.getDataSources(), ingestJobSettings);
/*
* Purge specific files to be tested.
*/
FileManager fileManager = openCase.getServices().getFileManager();
List<AbstractFile> results = fileManager.findFiles("%%", "sqlcipher");
assertEquals("Unexpected number of SQLCipher results.", 15, results.size());
for (AbstractFile file : results) {
/*
* Process only non-slack files.
*/
if (file.isFile() && !file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)) {
/*
* Determine which assertions to use for the file based on
* its name.
*/
List<BlackboardArtifact> artifactsList = file.getAllArtifacts();
String[] splitNameArray = file.getName().split("\\.");
if (splitNameArray[0].startsWith("sqlcipher-") && splitNameArray[splitNameArray.length - 1].equals("db")) {
/*
* Check that the SQLCipher database file has one
* TSK_ENCRYPTION_SUSPECTED artifact.
*/
int artifactsListSize = artifactsList.size();
String errorMessage = String.format("File '%s' (objId=%d) has %d artifacts, but 1 was expected.", file.getName(), file.getId(), artifactsListSize);
assertEquals(errorMessage, 1, artifactsListSize);
String artifactTypeName = artifactsList.get(0).getArtifactTypeName();
errorMessage = String.format("File '%s' (objId=%d) has an unexpected '%s' artifact.", file.getName(), file.getId(), artifactTypeName);
assertEquals(errorMessage, BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_SUSPECTED.toString(), artifactTypeName);
} else {
/*
* Check that the file has no artifacts.
*/
int artifactsListSize = artifactsList.size();
String errorMessage = String.format("File '%s' (objId=%d) has %d artifacts, but none were expected.", file.getName(), file.getId(), artifactsListSize);
assertEquals(errorMessage, 0, artifactsListSize);
}
}
}
} catch (TskCoreException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
}
Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/CaseUtils.java
-3
View File
@@ -38,8 +38,6 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
*/
public final class CaseUtils {
private static final String PRESERVE_CASE_DATA_LIST_FILE_NAME = ".preserve";
/**
* Create a case case directory and case for the given case name.
*
@@ -127,5 +125,4 @@ public final class CaseUtils {
*/
private CaseUtils() {
}
}