From 0d166feaa2e4ece5e3883d13adb74ab31ea769d1 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Wed, 2 May 2018 18:08:55 -0600 Subject: [PATCH] testing --- .../AllDataSourcesCommonFilesAlgorithm.java | 4 +- .../commonfilesearch/CommonFilesMetadata.java | 4 +- .../CommonFilesMetadataBuilder.java | 2 +- .../commonfilesearch/CommonFilesPanel.java | 55 +----- .../commonfilesearch/DataSourceLoader.java | 81 ++++++++ .../commonfilesearch/SingleDataSource.java | 10 +- .../IntraCaseCommonFilesSearchTest.java | 176 +++++++++++++----- 7 files changed, 223 insertions(+), 109 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/commonfilesearch/DataSourceLoader.java diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSourcesCommonFilesAlgorithm.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSourcesCommonFilesAlgorithm.java index 437d4d1885..a1b1af6aa4 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSourcesCommonFilesAlgorithm.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSourcesCommonFilesAlgorithm.java @@ -24,7 +24,7 @@ import java.util.Map; /** * Provides logic for selecting common files from all data sources. */ -final class AllDataSourcesCommonFilesAlgorithm extends CommonFilesMetadataBuilder { +final public class AllDataSourcesCommonFilesAlgorithm extends CommonFilesMetadataBuilder { private static final String WHERE_CLAUSE = "%s md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL)%s GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; //NON-NLS @@ -36,7 +36,7 @@ final class AllDataSourcesCommonFilesAlgorithm extends CommonFilesMetadataBuilde * @param filterByMediaMimeType match only on files whose mime types can be broadly categorized as media types * @param filterByDocMimeType match only on files whose mime types can be broadly categorized as document types */ - AllDataSourcesCommonFilesAlgorithm(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) { + public AllDataSourcesCommonFilesAlgorithm(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) { super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType); } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadata.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadata.java index c7c70676b8..1b405e9224 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadata.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadata.java @@ -27,7 +27,7 @@ import java.util.Map; * results. Subclass this to implement different selections of files from the * case. */ -final class CommonFilesMetadata { +final public class CommonFilesMetadata { private final Map metadata; @@ -61,7 +61,7 @@ final class CommonFilesMetadata { * How many distinct file instances exist for this metadata? * @return number of file instances */ - int size() { + public int size() { int count = 0; for (Md5Metadata data : this.metadata.values()) { count += data.size(); diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java index 2d279179ee..49525e7343 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java @@ -46,7 +46,7 @@ import org.sleuthkit.datamodel.TskCoreException; * This entire thing runs on a background thread where exceptions are handled. */ @SuppressWarnings("PMD.AbstractNaming") -abstract class CommonFilesMetadataBuilder { +public abstract class CommonFilesMetadataBuilder { private final Map dataSourceIdToNameMap; private final boolean filterByMedia; diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java index ad278a4cb5..866a76f061 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java @@ -42,7 +42,6 @@ import org.sleuthkit.autopsy.corecomponents.TableFilterNode; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.directorytree.DataResultFilterNode; -import org.sleuthkit.autopsy.directorytree.DirectoryTreeTopComponent; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbQuery; import org.sleuthkit.datamodel.TskCoreException; @@ -77,7 +76,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel { initComponents(); this.setupDataSources(); - + this.errorText.setVisible(false); } @@ -98,10 +97,6 @@ public final class CommonFilesPanel extends javax.swing.JPanel { new SwingWorker, Void>() { - private static final String SELECT_DATA_SOURCES_LOGICAL = "select obj_id, name from tsk_files where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))"; - - private static final String SELECT_DATA_SOURCES_IMAGE = "select obj_id, name from tsk_image_names where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))"; - private void updateUi() { String[] dataSourcesNames = new String[CommonFilesPanel.this.dataSourceMap.size()]; @@ -132,48 +127,10 @@ public final class CommonFilesPanel extends javax.swing.JPanel { return CommonFilesPanel.this.dataSourceMap.size() >= 2; } - private void loadLogicalSources(SleuthkitCase tskDb, Map dataSouceMap) throws TskCoreException, SQLException { - //try block releases resources - exceptions are handled in done() - try ( - CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_LOGICAL); - ResultSet resultSet = query.getResultSet()) { - while (resultSet.next()) { - Long objectId = resultSet.getLong(1); - String dataSourceName = resultSet.getString(2); - dataSouceMap.put(objectId, dataSourceName); - } - } - } - - private void loadImageSources(SleuthkitCase tskDb, Map dataSouceMap) throws SQLException, TskCoreException { - //try block releases resources - exceptions are handled in done() - try ( - CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_IMAGE); - ResultSet resultSet = query.getResultSet()) { - - while (resultSet.next()) { - Long objectId = resultSet.getLong(1); - String dataSourceName = resultSet.getString(2); - File image = new File(dataSourceName); - String dataSourceNameTrimmed = image.getName(); - dataSouceMap.put(objectId, dataSourceNameTrimmed); - } - } - } - @Override protected Map doInBackground() throws NoCurrentCaseException, TskCoreException, SQLException { - - Map dataSouceMap = new HashMap<>(); - - Case currentCase = Case.getCurrentCaseThrows(); - SleuthkitCase tskDb = currentCase.getSleuthkitCase(); - - loadLogicalSources(tskDb, dataSouceMap); - - loadImageSources(tskDb, dataSouceMap); - - return dataSouceMap; + DataSourceLoader loader = new DataSourceLoader(); + return loader.getDataSourceMap(); } @Override @@ -296,10 +253,10 @@ public final class CommonFilesPanel extends javax.swing.JPanel { TableFilterNode tableFilterWithDescendantsNode = new TableFilterNode(dataResultFilterNode); DataResultViewerTable table = new DataResultViewerTable(); - + Collection viewers = new ArrayList<>(1); viewers.add(table); - + DataResultTopComponent.createInstance(tabTitle, pathText, tableFilterWithDescendantsNode, metadata.size(), viewers); } catch (InterruptedException ex) { @@ -591,7 +548,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel { this.pictureVideoCheckbox.setEnabled(true); this.documentsCheckbox.setEnabled(true); - + this.toggleErrorTextAndSearchBox(); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/DataSourceLoader.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/DataSourceLoader.java new file mode 100644 index 0000000000..117d45bc90 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/DataSourceLoader.java @@ -0,0 +1,81 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilesearch; + +import java.io.File; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +public class DataSourceLoader { + + private static final String SELECT_DATA_SOURCES_LOGICAL = "select obj_id, name from tsk_files where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))"; + + private static final String SELECT_DATA_SOURCES_IMAGE = "select obj_id, name from tsk_image_names where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))"; + + public DataSourceLoader() { + } + + private void loadLogicalSources(SleuthkitCase tskDb, Map dataSouceMap) throws TskCoreException, SQLException { + //try block releases resources - exceptions are handled in done() + try (SleuthkitCase.CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_LOGICAL)) { + ResultSet resultSet = query.getResultSet(); + while (resultSet.next()) { + Long objectId = resultSet.getLong(1); + String dataSourceName = resultSet.getString(2); + dataSouceMap.put(objectId, dataSourceName); + } + } + } + + private void loadImageSources(SleuthkitCase tskDb, Map dataSouceMap) throws SQLException, TskCoreException { + //try block releases resources - exceptions are handled in done() + try ( + SleuthkitCase.CaseDbQuery query = tskDb.executeQuery(SELECT_DATA_SOURCES_IMAGE); + ResultSet resultSet = query.getResultSet()) { + + while (resultSet.next()) { + Long objectId = resultSet.getLong(1); + String dataSourceName = resultSet.getString(2); + File image = new File(dataSourceName); + String dataSourceNameTrimmed = image.getName(); + dataSouceMap.put(objectId, dataSourceNameTrimmed); + } + } + } + + public Map getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException { + Map dataSouceMap = new HashMap<>(); + + Case currentCase = Case.getOpenCase(); + SleuthkitCase tskDb = currentCase.getSleuthkitCase(); + + loadLogicalSources(tskDb, dataSouceMap); + + loadImageSources(tskDb, dataSouceMap); + + return dataSouceMap; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java index a3643d3306..5e31754c62 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java @@ -24,7 +24,7 @@ import java.util.Map; /** * Provides logic for selecting common files from a single data source. */ -final class SingleDataSource extends CommonFilesMetadataBuilder { +final public class SingleDataSource extends CommonFilesMetadataBuilder { private static final String WHERE_CLAUSE = "%s md5 in (select md5 from tsk_files where md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL) and data_source_obj_id=%s%s) GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; //NON-NLS private final Long selectedDataSourceId; @@ -37,10 +37,12 @@ final class SingleDataSource extends CommonFilesMetadataBuilder { * @param dataSourceId data source id for which common files must appear at * least once * @param dataSourceIdMap a map of obj_id to datasource name - * @param filterByMediaMimeType match only on files whose mime types can be broadly categorized as media types - * @param filterByDocMimeType match only on files whose mime types can be broadly categorized as document types + * @param filterByMediaMimeType match only on files whose mime types can be + * broadly categorized as media types + * @param filterByDocMimeType match only on files whose mime types can be + * broadly categorized as document types */ - SingleDataSource(Long dataSourceId, Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) { + public SingleDataSource(Long dataSourceId, Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) { super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType); this.selectedDataSourceId = dataSourceId; this.dataSourceName = dataSourceIdMap.get(this.selectedDataSourceId); diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseCommonFilesSearchTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseCommonFilesSearchTest.java index f9f744f200..4a09369193 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseCommonFilesSearchTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseCommonFilesSearchTest.java @@ -19,75 +19,149 @@ */ package org.sleuthkit.autopsy.commonfilessearch; -import java.io.File; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import static junit.framework.Assert.assertFalse; -import org.apache.commons.io.FileUtils; +import java.sql.SQLException; +import java.util.Map; +import junit.framework.Assert; +import static junit.framework.Assert.*; import org.netbeans.junit.NbTestCase; import org.openide.util.Exceptions; -import org.python.icu.impl.Assert; +import org.sleuthkit.autopsy.casemodule.ImageDSProcessor; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder; +import org.sleuthkit.autopsy.commonfilesearch.DataSourceLoader; +import org.sleuthkit.autopsy.commonfilesearch.SingleDataSource; import org.sleuthkit.autopsy.testutils.CaseUtils; +import org.sleuthkit.autopsy.testutils.IngestUtils; +import org.sleuthkit.datamodel.TskCoreException; /** * * @author bsweeney */ -public class IntraCaseCommonFilesSearchTest extends NbTestCase { - +public abstract class IntraCaseCommonFilesSearchTest extends NbTestCase { + private static final String CASE_NAME = "IntraCaseCommonFilesSearchTest"; private static final Path CASE_DIRECTORY_PATH = Paths.get(System.getProperty("java.io.tmpdir"), CASE_NAME); - private static final File CASE_DIR = new File(CASE_DIRECTORY_PATH.toString()); - private final Path IMAGE_PATH_1 = Paths.get(this.getDataDir().toString(), "3776", "3776-1.e01.ad1"); - private final Path IMAGE_PATH_2 = Paths.get(this.getDataDir().toString(), "3776", "3776-2.e01.ad1"); - private final Path IMAGE_PATH_3 = Paths.get(this.getDataDir().toString(), "3776", "3776-3.e01.ad1"); - private final Path IMAGE_PATH_4 = Paths.get(this.getDataDir().toString(), "3776", "3776-4.e01.ad1"); - + + private final Path IMAGE_PATH_1 = Paths.get(this.getDataDir().toString(), "3776", "commonfiles_image1_v1.vhd"); + private final Path IMAGE_PATH_2 = Paths.get(this.getDataDir().toString(), "3776", "commonfiles_image2_v1.vhd"); + private final Path IMAGE_PATH_3 = Paths.get(this.getDataDir().toString(), "3776", "commonfiles_image3_v1.vhd"); + private final Path IMAGE_PATH_4 = Paths.get(this.getDataDir().toString(), "3776", "commonfiles_image4_v1.vhd"); + + protected DataSourceLoader dataSourceLoader; + public IntraCaseCommonFilesSearchTest(String name) { super(name); } - - @Override - public void setUp(){ - CaseUtils.createCase(CASE_DIRECTORY_PATH); - } - - /** - * Add images #1, #2, #3, and #4 to case. Do not ingest. - * Find all matches & all file types. Confirm no matches are found (since there are no hashes to match). - * Find all matches on image #1 & all file types. Confirm no matches. - */ - public void testOne(){ - - } - - - /** - * Add #1, #2, #3, and #4 to case and ingest with hash algorithm. - * Find all matches & all file types. Confirm file.jpg is found on all three and file.docx is found on two. - * Find matches on ‘#1’ & all file types. Confirm same results. - * Find matches on ‘#2 & all file types: Confirm file.jpg. - * Find matches on ‘#3’ & all file types: Confirm file.jpg and file.docx. - * Find matches on #4 & all file types: Confirm nothing is found - */ - public void testTwo(){ - - } - - /** - * Add #1 and #4 to case and ingest. - * Find all matches & all file types. Confirm nothing matches - */ - public void testThree(){ - - } - @Override - public void tearDown(){ + public void setUp() { + + CaseUtils.createCase(CASE_DIRECTORY_PATH); + + IngestUtils.addDataSource(new ImageDSProcessor(), IMAGE_PATH_1); + IngestUtils.addDataSource(new ImageDSProcessor(), IMAGE_PATH_2); + IngestUtils.addDataSource(new ImageDSProcessor(), IMAGE_PATH_3); + IngestUtils.addDataSource(new ImageDSProcessor(), IMAGE_PATH_4); + + this.dataSourceLoader = new DataSourceLoader(); + } + + @Override + public void tearDown() { CaseUtils.closeCase(); CaseUtils.deleteCaseDir(CASE_DIRECTORY_PATH); } - + + public class UningestedCases extends IntraCaseCommonFilesSearchTest { + + public UningestedCases(String name) { + super(name); + } + + /** + * Add images #1, #2, #3, and #4 to case. Do not ingest. Find all + * matches & all file types. Confirm no matches are found (since there + * are no hashes to match). Find all matches on image #1 & all file + * types. Confirm no matches. + */ + public void testOne() { + try { + Map dataSources = this.dataSourceLoader.getDataSourceMap(); + + CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false); + CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles(); + + int resultCount = metadata.size(); + assertEquals(resultCount, 0); + + } catch (NoCurrentCaseException | TskCoreException | SQLException ex) { + fail(ex.getMessage()); + } + } + + public void testTwo() { + try { + Map dataSources = this.dataSourceLoader.getDataSourceMap(); + Long first = new Long(1); + + CommonFilesMetadataBuilder singleSourceBuilder = new SingleDataSource(first, dataSources, false, false); + CommonFilesMetadata metadata = singleSourceBuilder.findCommonFiles(); + + int resultCount = metadata.size(); + assertEquals(resultCount, 0); + + } catch (NoCurrentCaseException | TskCoreException | SQLException ex) { + fail(ex.getMessage()); + } + } + } + + public class IngestedWithHashAlgOnly extends IntraCaseCommonFilesSearchTest { + + public IngestedWithHashAlgOnly(String name) { + super(name); + } + + /** + * Add #1, #2, #3, and #4 to case and ingest with hash algorithm. Find + * all matches & all file types. Confirm file.jpg is found on all three + * and file.docx is found on two. Find matches on ‘#1’ & all file types. + * Confirm same results. Find matches on ‘#2 & all file types: Confirm + * file.jpg. Find matches on ‘#3’ & all file types: Confirm file.jpg and + * file.docx. Find matches on #4 & all file types: Confirm nothing is + * found + */ + public void testTwo() { + + } + + } + + public class NoMatches extends IntraCaseCommonFilesSearchTest { + + public NoMatches(String name) { + super(name); + } + + @Override + public void setUp() { + + CaseUtils.createCase(CASE_DIRECTORY_PATH); + IngestUtils.addDataSource(new ImageDSProcessor(), IMAGE_PATH_1); + IngestUtils.addDataSource(new ImageDSProcessor(), IMAGE_PATH_4); + } + + /** + * Add #1 and #4 to case and ingest. Find all matches & all file types. + * Confirm nothing matches + */ + public void testThree() { + + } + } }