From 48e1f224c96dc8079e2d2c847af60d2ffb92bbf4 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 11 Mar 2019 13:56:38 -0400 Subject: [PATCH] Moved some of the Experimental utility code to Autopsy Core --- .../AddDataSourceCallback.java | 10 +- .../AutoIngestDataSource.java | 20 +- .../DataSourceProcessorUtility.java | 15 +- .../autoingest/AddArchiveTask.java | 403 ------------------ .../ArchiveExtractorDSProcessor.java | 176 -------- .../autoingest/ArchiveFilePanel.form | 94 ---- .../autoingest/ArchiveFilePanel.java | 289 ------------- .../experimental/autoingest/ArchiveUtil.java | 313 -------------- .../autoingest/AutoIngestManager.java | 5 +- .../experimental/autoingest/Bundle.properties | 4 - .../autoingest/Bundle.properties-MERGED | 10 - 11 files changed, 25 insertions(+), 1314 deletions(-) rename {Experimental/src/org/sleuthkit/autopsy/experimental/autoingest => Core/src/org/sleuthkit/autopsy/datasourceprocessors}/AddDataSourceCallback.java (90%) rename {Experimental/src/org/sleuthkit/autopsy/experimental/autoingest => Core/src/org/sleuthkit/autopsy/datasourceprocessors}/AutoIngestDataSource.java (71%) rename {Experimental/src/org/sleuthkit/autopsy/experimental/autoingest => Core/src/org/sleuthkit/autopsy/datasourceprocessors}/DataSourceProcessorUtility.java (83%) delete mode 100644 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java delete mode 100644 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java delete mode 100644 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form delete mode 100644 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java delete mode 100644 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AddDataSourceCallback.java similarity index 90% rename from Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java rename to Core/src/org/sleuthkit/autopsy/datasourceprocessors/AddDataSourceCallback.java index fc00149249..3ee02b9026 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AddDataSourceCallback.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.experimental.autoingest; +package org.sleuthkit.autopsy.datasourceprocessors; import java.util.List; import java.util.UUID; @@ -32,7 +32,7 @@ import org.sleuthkit.datamodel.Content; * processor finishes running in its own thread. */ @Immutable -class AddDataSourceCallback extends DataSourceProcessorCallback { +public class AddDataSourceCallback extends DataSourceProcessorCallback { private final Case caseForJob; private final AutoIngestDataSource dataSourceInfo; @@ -48,7 +48,7 @@ class AddDataSourceCallback extends DataSourceProcessorCallback { * @param dataSourceInfo The data source * @param taskId The task id to associate with ingest job events. */ - AddDataSourceCallback(Case caseForJob, AutoIngestDataSource dataSourceInfo, UUID taskId, Object lock) { + public AddDataSourceCallback(Case caseForJob, AutoIngestDataSource dataSourceInfo, UUID taskId, Object lock) { this.caseForJob = caseForJob; this.dataSourceInfo = dataSourceInfo; this.taskId = taskId; @@ -87,7 +87,7 @@ class AddDataSourceCallback extends DataSourceProcessorCallback { * @param result The result code for the processing of the data source. * @param errorMessages Any error messages generated during the processing * of the data source. - * @param dataSourceContent The content produced by processing the data + * @param dataSources The content produced by processing the data * source. */ @Override diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDataSource.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSource.java similarity index 71% rename from Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDataSource.java rename to Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSource.java index 89a0d0ad5f..fbf5de11c5 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDataSource.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSource.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.experimental.autoingest; +package org.sleuthkit.autopsy.datasourceprocessors; import java.nio.file.Path; import java.util.ArrayList; @@ -26,7 +26,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback import org.sleuthkit.datamodel.Content; @ThreadSafe -class AutoIngestDataSource { +public class AutoIngestDataSource { private final String deviceId; private final Path path; @@ -34,34 +34,34 @@ class AutoIngestDataSource { private List errorMessages; private List content; - AutoIngestDataSource(String deviceId, Path path) { + public AutoIngestDataSource(String deviceId, Path path) { this.deviceId = deviceId; this.path = path; } - String getDeviceId() { + public String getDeviceId() { return deviceId; } - Path getPath() { + public Path getPath() { return this.path; } - synchronized void setDataSourceProcessorOutput(DataSourceProcessorResult result, List errorMessages, List content) { + public synchronized void setDataSourceProcessorOutput(DataSourceProcessorResult result, List errorMessages, List content) { this.resultCode = result; this.errorMessages = new ArrayList<>(errorMessages); this.content = new ArrayList<>(content); } - synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode() { + public synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode() { return resultCode; } - synchronized List getDataSourceProcessorErrorMessages() { + public synchronized List getDataSourceProcessorErrorMessages() { return new ArrayList<>(errorMessages); } - synchronized List getContent() { + public synchronized List getContent() { return new ArrayList<>(content); } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSourceProcessorUtility.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/DataSourceProcessorUtility.java similarity index 83% rename from Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSourceProcessorUtility.java rename to Core/src/org/sleuthkit/autopsy/datasourceprocessors/DataSourceProcessorUtility.java index 4878f7fa7d..8b27ff6c19 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSourceProcessorUtility.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/DataSourceProcessorUtility.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.experimental.autoingest; +package org.sleuthkit.autopsy.datasourceprocessors; import java.nio.file.Path; import java.util.Collection; @@ -25,13 +25,12 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openide.util.Lookup; -import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; /** * A utility class to find Data Source Processors */ -class DataSourceProcessorUtility { +public class DataSourceProcessorUtility { private DataSourceProcessorUtility() { } @@ -47,7 +46,7 @@ class DataSourceProcessorUtility { * @throws * org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException */ - static Map getDataSourceProcessorForFile(Path dataSourcePath, Collection processorCandidates) throws AutoIngestDataSourceProcessorException { + public static Map getDataSourceProcessorForFile(Path dataSourcePath, Collection processorCandidates) throws AutoIngestDataSourceProcessorException { Map validDataSourceProcessorsMap = new HashMap<>(); for (AutoIngestDataSourceProcessor processor : processorCandidates) { int confidence = processor.canProcess(dataSourcePath); @@ -74,7 +73,7 @@ class DataSourceProcessorUtility { * @throws * org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException */ - static List getOrderedListOfDataSourceProcessors(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { + public static List getOrderedListOfDataSourceProcessors(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { // lookup all AutomatedIngestDataSourceProcessors Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); return getOrderedListOfDataSourceProcessors(dataSourcePath, processorCandidates); @@ -96,7 +95,7 @@ class DataSourceProcessorUtility { * @throws * org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException */ - static List getOrderedListOfDataSourceProcessors(Path dataSourcePath, Collection processorCandidates) throws AutoIngestDataSourceProcessorException { + public static List getOrderedListOfDataSourceProcessors(Path dataSourcePath, Collection processorCandidates) throws AutoIngestDataSourceProcessorException { Map validDataSourceProcessorsMap = getDataSourceProcessorForFile(dataSourcePath, processorCandidates); return orderDataSourceProcessorsByConfidence(validDataSourceProcessorsMap); } @@ -110,7 +109,7 @@ class DataSourceProcessorUtility { * the data source along with their confidence score * @return Ordered list of data source processors */ - static List orderDataSourceProcessorsByConfidence(Map validDataSourceProcessorsMap) { + public static List orderDataSourceProcessorsByConfidence(Map validDataSourceProcessorsMap) { List validDataSourceProcessors = validDataSourceProcessorsMap.entrySet().stream() .sorted(Map.Entry.comparingByValue().reversed()) .map(Map.Entry::getKey) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java deleted file mode 100644 index 93c30fdb53..0000000000 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.experimental.autoingest; - -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; -import java.util.logging.Level; -import java.util.stream.Collectors; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.openide.util.Lookup; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor; -import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; -import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; -import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; -import org.sleuthkit.autopsy.coreutils.TimeStampUtils; -import org.sleuthkit.autopsy.datasourceprocessors.RawDSProcessor; -import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DataSource; -import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.VirtualDirectory; - -/* - * A runnable that adds an archive data source as well as data sources contained - * in the archive to the case database. - */ -class AddArchiveTask implements Runnable { - - private final Logger logger = Logger.getLogger(AddArchiveTask.class.getName()); - private final String deviceId; - private final String archivePath; - private final DataSourceProcessorProgressMonitor progressMonitor; - private final DataSourceProcessorCallback callback; - private boolean criticalErrorOccurred; - private final Object archiveDspLock; - - private static final String ARCHIVE_EXTRACTOR_MODULE_OUTPUT_DIR = "Archive Extractor"; - private static final String LOGICAL_FILE_VIRTUAL_DIR_NAME = "$AdditionalFiles"; - - /** - * Constructs a runnable task that adds an archive as well as data sources - * contained in the archive to the case database. - * - * @param deviceId An ASCII-printable identifier for the device - * associated with the data source that is intended - * to be unique across multiple cases (e.g., a UUID). - * @param archivePath Path to the archive file. - * @param progressMonitor Progress monitor to report progress during - * processing. - * @param callback Callback to call when processing is done. - */ - AddArchiveTask(String deviceId, String archivePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - this.deviceId = deviceId; - this.archivePath = archivePath; - this.callback = callback; - this.progressMonitor = progressMonitor; - this.archiveDspLock = new Object(); - } - - /** - * Adds the archive to the case database. - */ - @Override - public void run() { - progressMonitor.setIndeterminate(true); - List errorMessages = new ArrayList<>(); - List newDataSources = new ArrayList<>(); - DataSourceProcessorCallback.DataSourceProcessorResult result; - if (!ArchiveUtil.isArchive(Paths.get(archivePath))) { - criticalErrorOccurred = true; - logger.log(Level.SEVERE, String.format("Input data source is not a valid datasource: %s", archivePath)); //NON-NLS - errorMessages.add("Input data source is not a valid datasource: " + archivePath); - result = DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; - callback.done(result, errorMessages, newDataSources); - } - - logger.log(Level.INFO, "Using Archive Extractor DSP to process archive {0} ", archivePath); - - // extract the archive and pass the extracted folder as input - try { - Case currentCase = Case.getCurrentCaseThrows(); - - // create folder to extract archive to - Path destinationFolder = createDirectoryForFile(archivePath, currentCase.getModuleDirectory()); - if (destinationFolder.toString().isEmpty()) { - // unable to create directory - criticalErrorOccurred = true; - errorMessages.add(String.format("Unable to create directory {0} to extract archive {1} ", new Object[]{destinationFolder.toString(), archivePath})); - logger.log(Level.SEVERE, "Unable to create directory {0} to extract archive {1} ", new Object[]{destinationFolder.toString(), archivePath}); - return; - } - - // extract contents of ZIP archive into destination folder - List extractedFiles = new ArrayList<>(); - try { - progressMonitor.setProgressText(String.format("Extracting archive contents to: %s", destinationFolder.toString())); - extractedFiles = ArchiveUtil.unpackArchiveFile(archivePath, destinationFolder.toString()); - } catch (ArchiveUtil.ArchiveExtractionException ex) { - // delete extracted contents - logger.log(Level.SEVERE, "Exception while extracting archive contents into " + destinationFolder.toString() + ". Deleteing the directory", ex); - FileUtils.deleteDirectory(destinationFolder.toFile()); - throw ex; - } - - List unclaimedFiles = new ArrayList<>(extractedFiles); - - // lookup all AutomatedIngestDataSourceProcessors so that we only do it once. - // LocalDisk, LocalFiles, and ArchiveDSP are removed from the list. - List processorCandidates = getListOfValidDataSourceProcessors(); - - // do processing - int numValidDataSources = 0; - DataSource defaultDataSource = null; - for (String file : extractedFiles) { - - // we only care about files, skip directories - File fileObject = new File(file); - if (fileObject.isDirectory()) { - continue; - } - - // identify all "valid" DSPs that can process this file - List validDataSourceProcessors = getDataSourceProcessorsForFile(Paths.get(file), errorMessages, processorCandidates); - if (validDataSourceProcessors.isEmpty()) { - continue; - } - - // identified a "valid" data source within the archive - progressMonitor.setProgressText(String.format("Adding: %s", file)); - - /* - * NOTE: we have to move the valid data sources to a separate - * folder and then add the data source from that folder. This is - * necessary because after all valid data sources have been - * identified, we are going to add the remaining extracted - * contents of the archive as a single logical file set. Hence, - * if we do not move the data sources out of the extracted - * contents folder, those data source files will get added twice - * and can potentially result in duplicate keyword hits. - - Path newFolder = createDirectoryForFile(file, currentCase.getModuleDirectory()); - if (newFolder.toString().isEmpty()) { - // unable to create directory - criticalErrorOccurred = true; - errorMessages.add(String.format("Unable to create directory {0} to extract content of archive {1} ", new Object[]{newFolder.toString(), archivePath})); - logger.log(Level.SEVERE, "Unable to create directory {0} to extract content of archive {1} ", new Object[]{newFolder.toString(), archivePath}); - return; - } - - // Copy it to a different folder - //FileUtils.copyFileToDirectory(fileObject, newFolder.toFile()); - //Path newFilePath = Paths.get(newFolder.toString(), FilenameUtils.getName(file)); -*/ - // Try each DSP in decreasing order of confidence - boolean success = false; - for (AutoIngestDataSourceProcessor selectedProcessor : validDataSourceProcessors) { - - logger.log(Level.INFO, "Using {0} to process extracted file {1} ", new Object[]{selectedProcessor.getDataSourceType(), file}); - synchronized (archiveDspLock) { - UUID taskId = UUID.randomUUID(); - currentCase.notifyAddingDataSource(taskId); - AutoIngestDataSource internalDataSource = new AutoIngestDataSource(deviceId, fileObject.toPath()); - DataSourceProcessorCallback internalArchiveDspCallBack = new AddDataSourceCallback(currentCase, internalDataSource, taskId, archiveDspLock); - selectedProcessor.process(deviceId, fileObject.toPath(), progressMonitor, internalArchiveDspCallBack); - archiveDspLock.wait(); - - // at this point we got the content object(s) from the current DSP. - // check whether the data source was processed successfully - if ((internalDataSource.getResultDataSourceProcessorResultCode() == CRITICAL_ERRORS) - || internalDataSource.getContent().isEmpty()) { - // move onto the the next DSP that can process this data source - for (String errorMessage : internalDataSource.getDataSourceProcessorErrorMessages()) { - logger.log(Level.SEVERE, "Data source processor {0} was unable to process {1}: {2}", new Object[]{selectedProcessor.getDataSourceType(), internalDataSource.getPath(), errorMessage}); - } - continue; - } - - // if we are here it means the data source was added successfully - success = true; - newDataSources.addAll(internalDataSource.getContent()); - numValidDataSources++; - - // this extracted file has been "claimed" by one of the DSPs, - // remove it from the list of Logical Files that will be added later - removeClaimedFiles(file, unclaimedFiles, selectedProcessor); - - // update data source info - for (Content c:internalDataSource.getContent()) { - if (c instanceof DataSource) { - DataSource ds = (DataSource) c; - - // Read existing aquisition details and update them - String details = "Extracted from archive: " + archivePath.toString(); - String existingDetails = ds.getAcquisitionDetails(); - if (existingDetails != null && !existingDetails.isEmpty()) { - ds.setAcquisitionDetails(existingDetails + System.getProperty("line.separator") + details); - } else { - ds.setAcquisitionDetails(details); - } - - // Update the names for all new data sources to be the root archive plus the name of the data source - String newName = Paths.get(archivePath).getFileName() + "/" + ds.getName(); - ds.setDisplayName(newName); - currentCase.notifyDataSourceNameChanged(c, newName); - - defaultDataSource = ds; - } - } - - // skip all other DSPs for this data source - break; - } - } - - /*if (success) { - // one of the DSPs successfully processed the data source. delete the - // copy of the data source in the original extracted archive folder. - // otherwise the data source is going to be added again as a logical file. - numValidDataSources--; - FileUtils.deleteQuietly(fileObject); - } else { - // none of the DSPs were able to process the data source. delete the - // copy of the data source in the temporary folder. the data source is - // going to be added as a logical file with the rest of the extracted contents. - FileUtils.deleteQuietly(newFolder.toFile()); - }*/ - } - - // after all archive contents have been examined (and moved to separate folders if necessary), - // add remaining extracted contents as one logical file set - if (unclaimedFiles.size() > 0) { - - // ELTODO investigate if i need to aquire datase lock? - SleuthkitCase caseDatabase = Case.getCurrentCaseThrows().getSleuthkitCase(); - VirtualDirectory additionalFilesDir = caseDatabase.addVirtualDirectory(defaultDataSource.getId(), LOGICAL_FILE_VIRTUAL_DIR_NAME); - - for (String file : unclaimedFiles) { - File fileObject = new File(file); - caseDatabase.addLocalFile(fileObject.getName(), fileObject.getAbsolutePath(), fileObject.length(), 0, 0, 0, 0, fileObject.isFile(), TskData.EncodingType.NONE, additionalFilesDir); - } - - /*progressMonitor.setProgressText(String.format("Adding: %s", destinationFolder.toString())); - logger.log(Level.INFO, "Adding directory {0} as logical file set", destinationFolder.toString()); - synchronized (archiveDspLock) { - UUID taskId = UUID.randomUUID(); - currentCase.notifyAddingDataSource(taskId); - AutoIngestDataSource internalDataSource = new AutoIngestDataSource(deviceId, destinationFolder); - DataSourceProcessorCallback internalArchiveDspCallBack = new AddDataSourceCallback(currentCase, internalDataSource, taskId, archiveDspLock); - - // folder where archive was extracted to - List pathsList = new ArrayList<>(); - pathsList.add(destinationFolder.toString()); - - // use archive file name as the name of the logical file set - String archiveFileName = FilenameUtils.getName(archivePath); - - LocalFilesDSProcessor localFilesDSP = new LocalFilesDSProcessor(); - localFilesDSP.run(deviceId, archiveFileName, unclaimedFiles, progressMonitor, internalArchiveDspCallBack); - - archiveDspLock.wait(); - - // at this point we got the content object(s) from the current DSP. - newDataSources.addAll(internalDataSource.getContent()); - - for (Content c : internalDataSource.getContent()) { - if (c instanceof DataSource) { - DataSource ds = (DataSource) c; - // This is a new data source so just write the aquisition details - String details = "Extracted from archive: " + archivePath.toString(); - ds.setAcquisitionDetails(details); - } - } - }*/ - } - } catch (Exception ex) { - criticalErrorOccurred = true; - errorMessages.add(ex.getMessage()); - logger.log(Level.SEVERE, String.format("Critical error occurred while extracting archive %s", archivePath), ex); //NON-NLS - } finally { - logger.log(Level.INFO, "Finished processing of archive {0}", archivePath); - progressMonitor.setProgress(100); - if (criticalErrorOccurred) { - result = DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; - } else if (!errorMessages.isEmpty()) { - result = DataSourceProcessorCallback.DataSourceProcessorResult.NONCRITICAL_ERRORS; - } else { - result = DataSourceProcessorCallback.DataSourceProcessorResult.NO_ERRORS; - } - callback.done(result, errorMessages, newDataSources); - } - } - - private void removeClaimedFiles(String file, List unclaimedFiles, AutoIngestDataSourceProcessor selectedProcessor) { - - //if (!(selectedProcessor instanceof ForensicToolReportProcessor)) { - unclaimedFiles.remove(file); - //} - } - - /** - * Get a list of data source processors. LocalFiles, RawDSProcessor, and - * ArchiveDSP are removed from the list. - * - * @return List of data source processors - */ - private List getListOfValidDataSourceProcessors() { - - Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); - - List validDataSourceProcessors = processorCandidates.stream().collect(Collectors.toList()); - - for (Iterator iterator = validDataSourceProcessors.iterator(); iterator.hasNext();) { - AutoIngestDataSourceProcessor selectedProcessor = iterator.next(); - - // skip local files, only looking for "valid" data sources. - // also skip RawDSP as we don't want to add random "bin" and "raw" files that may be inside archive - // as individual data sources. - // also skip nested archive files, those will be ingested as logical files and extracted during ingest - if ((selectedProcessor instanceof LocalFilesDSProcessor) - || (selectedProcessor instanceof RawDSProcessor) - || (selectedProcessor instanceof ArchiveExtractorDSProcessor)) { - iterator.remove(); - } - } - - return validDataSourceProcessors; - } - - /** - * Get a list of data source processors that can process the data source of - * interest. The list is sorted by confidence in decreasing order. - * - * @param dataSourcePath Full path to the data source - * @param errorMessages List for error messages - * @param errorMessages List of AutoIngestDataSourceProcessor to try - * - * @return Ordered list of applicable DSPs - */ - private List getDataSourceProcessorsForFile(Path dataSourcePath, List errorMessages, - List processorCandidates) { - - // Get an ordered list of data source processors to try - List validDataSourceProcessorsForFile = Collections.emptyList(); - try { - validDataSourceProcessorsForFile = DataSourceProcessorUtility.getOrderedListOfDataSourceProcessors(dataSourcePath, processorCandidates); - } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { - criticalErrorOccurred = true; - errorMessages.add(ex.getMessage()); - logger.log(Level.SEVERE, String.format("Critical error occurred while extracting archive %s", archivePath), ex); //NON-NLS - return Collections.emptyList(); - } - return validDataSourceProcessorsForFile; - } - - /** - * Create a directory in ModuleOutput folder based on input file name. A - * time stamp is appended to the directory name. - * - * @param fileName File name - * @param baseDirectory Base directory. Typically the case output directory. - * - * @return Full path to the new directory - */ - private Path createDirectoryForFile(String fileName, String baseDirectory) { - // get file name without full path or extension - String fileNameNoExt = FilenameUtils.getBaseName(fileName); - - // create folder to extract archive to - Path newFolder = Paths.get(baseDirectory, ARCHIVE_EXTRACTOR_MODULE_OUTPUT_DIR, fileNameNoExt + "_" + TimeStampUtils.createTimeStamp()); - if (newFolder.toFile().mkdirs() == false) { - // unable to create directory - return Paths.get(""); - } - return newFolder; - } -} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java deleted file mode 100644 index a30fbf0c75..0000000000 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011-2017 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.experimental.autoingest; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.nio.file.Path; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import javax.swing.JPanel; -import org.openide.util.NbBundle; -import org.openide.util.lookup.ServiceProvider; -import org.openide.util.lookup.ServiceProviders; -import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; -import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; -import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; - -/** - * A data source processor that handles archive files. Implements the - * DataSourceProcessor service provider interface to allow integration with the - * add data source wizard. It also provides a run method overload to allow it to - * be used independently of the wizard. - */ -@ServiceProviders(value={ - @ServiceProvider(service=AutoIngestDataSourceProcessor.class)} -) -@NbBundle.Messages({ - "ArchiveDSP.dsType.text=Archive file"}) -public class ArchiveExtractorDSProcessor implements AutoIngestDataSourceProcessor { - - private final static String DATA_SOURCE_TYPE = Bundle.ArchiveDSP_dsType_text(); - - private final ArchiveFilePanel configPanel; - private String deviceId; - private String archivePath; - private boolean setDataSourceOptionsCalled; - - private final ExecutorService jobProcessingExecutor; - private static final String ARCHIVE_DSP_THREAD_NAME = "Archive-DSP-%d"; - private AddArchiveTask addArchiveTask; - - /** - * Constructs an archive data source processor that - * implements the DataSourceProcessor service provider interface to allow - * integration with the add data source wizard. It also provides a run - * method overload to allow it to be used independently of the wizard. - */ - public ArchiveExtractorDSProcessor() { - configPanel = ArchiveFilePanel.createInstance(ArchiveExtractorDSProcessor.class.getName(), ArchiveUtil.getArchiveFilters()); - jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(ARCHIVE_DSP_THREAD_NAME).build()); - } - - @Override - public int canProcess(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { - // check whether this is an archive - if (ArchiveUtil.isArchive(dataSourcePath)){ - // return "high confidence" value - return 100; - } - return 0; - } - - @Override - public void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { - run(deviceId, dataSourcePath.toString(), progressMonitor, callBack); - } - - @Override - public String getDataSourceType() { - return DATA_SOURCE_TYPE; - } - - /** - * Gets the panel that allows a user to select a data source and do any - * configuration required by the data source. The panel is less than 544 - * pixels wide and less than 173 pixels high. - * - * @return A selection and configuration panel for this data source - * processor. - */ - @Override - public JPanel getPanel() { - configPanel.readSettings(); - configPanel.select(); - return configPanel; - } - - /** - * Indicates whether the settings in the selection and configuration panel - * are valid and complete. - * - * @return True if the settings are valid and complete and the processor is - * ready to have its run method called, false otherwise. - */ - @Override - public boolean isPanelValid() { - return configPanel.validatePanel(); - } - - /** - * Adds a data source to the case database using a background task in a - * separate thread and the settings provided by the selection and - * configuration panel. Returns as soon as the background task is started. - * The background task uses a callback object to signal task completion and - * return results. - * - * This method should not be called unless isPanelValid returns true. - * - * @param progressMonitor Progress monitor that will be used by the - * background task to report progress. - * @param callback Callback that will be used by the background task - * to return results. - */ - @Override - public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - if (!setDataSourceOptionsCalled) { - configPanel.storeSettings(); - deviceId = UUID.randomUUID().toString(); - archivePath = configPanel.getContentPaths(); - } - run(deviceId, archivePath, progressMonitor, callback); - } - - /** - * Adds a data source to the case database using a background task in a - * separate thread and the given settings instead of those provided by the - * selection and configuration panel. Returns as soon as the background task - * is started and uses the callback object to signal task completion and - * return results. - * - * @param deviceId An ASCII-printable identifier for the device - * associated with the data source that is - * intended to be unique across multiple cases - * (e.g., a UUID). - * @param archivePath Path to the archive file. - * @param progressMonitor Progress monitor for reporting progress - * during processing. - * @param callback Callback to call when processing is done. - */ - public void run(String deviceId, String archivePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - addArchiveTask = new AddArchiveTask(deviceId, archivePath, progressMonitor, callback); - jobProcessingExecutor.submit(addArchiveTask); - } - - /** - * This DSP is a service to AutoIngestDataSourceProcessor only. Hence it is - * only used by AIM. AIM currently doesn't support DSP cancellation. - */ - @Override - public void cancel() { - } - - @Override - public void reset() { - deviceId = null; - archivePath = null; - configPanel.reset(); - setDataSourceOptionsCalled = false; - } -} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form deleted file mode 100644 index af9347df0c..0000000000 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form +++ /dev/null @@ -1,94 +0,0 @@ - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java deleted file mode 100644 index cbe39cac1b..0000000000 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.experimental.autoingest; - -import java.io.File; -import java.util.List; -import java.util.logging.Level; -import javax.swing.JFileChooser; -import javax.swing.JPanel; -import javax.swing.event.DocumentEvent; -import javax.swing.event.DocumentListener; -import javax.swing.filechooser.FileFilter; -import org.apache.commons.lang3.StringUtils; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import static org.sleuthkit.autopsy.experimental.autoingest.Bundle.*; -import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; -import org.sleuthkit.autopsy.coreutils.DriveUtils; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.ModuleSettings; -import org.sleuthkit.autopsy.coreutils.PathValidator; - -/** - * Panel for adding an archive file which is supported by 7zip library (e.g. - * "zip", "rar", "arj", "7z", "7zip", "gzip, etc). Allows the user to select a - * file. - */ -@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives -class ArchiveFilePanel extends JPanel implements DocumentListener { - - private static final Logger logger = Logger.getLogger(ArchiveFilePanel.class.getName()); - private static final String PROP_LAST_ARCHIVE_PATH = "LBL_LastImage_PATH"; //NON-NLS - - private final JFileChooser fileChooser = new JFileChooser(); - - /** - * Externally supplied name is used to store settings - */ - private final String contextName; - - /** - * Creates new form ArchiveFilePanel - * - * @param context A string context name used to read/store last - * used settings. - * @param fileChooserFilters A list of filters to be used with the - * FileChooser. - */ - private ArchiveFilePanel(String context, List fileChooserFilters) { - this.contextName = context; - initComponents(); - - errorLabel.setVisible(false); - - fileChooser.setDragEnabled(false); - fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); - fileChooser.setMultiSelectionEnabled(false); - fileChooserFilters.forEach(fileChooser::addChoosableFileFilter); - if (fileChooserFilters.isEmpty() == false) { - fileChooser.setFileFilter(fileChooserFilters.get(0)); - } - } - - /** - * Creates and returns an instance of a ArchiveFilePanel. - * - * @param context A string context name used to read/store last - * used settings. - * @param fileChooserFilters A list of filters to be used with the - * FileChooser. - * - * @return instance of the ArchiveFilePanel - */ - public static synchronized ArchiveFilePanel createInstance(String context, List fileChooserFilters) { - ArchiveFilePanel instance = new ArchiveFilePanel(context, fileChooserFilters); - // post-constructor initialization of listener support without leaking references of uninitialized objects - instance.pathTextField.getDocument().addDocumentListener(instance); - return instance; - } - - /** - * This method is called from within the constructor to initialize the form. - * WARNING: Do NOT modify this code. The content of this method is always - * regenerated by the Form Editor. - */ - // //GEN-BEGIN:initComponents - private void initComponents() { - - pathLabel = new javax.swing.JLabel(); - browseButton = new javax.swing.JButton(); - pathTextField = new javax.swing.JTextField(); - errorLabel = new javax.swing.JLabel(); - - setMinimumSize(new java.awt.Dimension(0, 65)); - setPreferredSize(new java.awt.Dimension(403, 65)); - - org.openide.awt.Mnemonics.setLocalizedText(pathLabel, org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.pathLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(browseButton, org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.browseButton.text")); // NOI18N - browseButton.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - browseButtonActionPerformed(evt); - } - }); - - pathTextField.setText(org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.pathTextField.text")); // NOI18N - - errorLabel.setForeground(new java.awt.Color(255, 0, 0)); - org.openide.awt.Mnemonics.setLocalizedText(errorLabel, org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.errorLabel.text")); // NOI18N - - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); - this.setLayout(layout); - layout.setHorizontalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addComponent(pathTextField) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(browseButton) - .addGap(2, 2, 2)) - .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(pathLabel) - .addComponent(errorLabel)) - .addGap(0, 277, Short.MAX_VALUE)) - ); - layout.setVerticalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addComponent(pathLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(browseButton) - .addComponent(pathTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGap(3, 3, 3) - .addComponent(errorLabel) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) - ); - }// //GEN-END:initComponents - - private void browseButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_browseButtonActionPerformed - String oldText = getContentPaths(); - // set the current directory of the FileChooser if the ArchivePath Field is valid - File currentDir = new File(oldText); - if (currentDir.exists()) { - fileChooser.setCurrentDirectory(currentDir); - } - - if (fileChooser.showOpenDialog(this) == JFileChooser.APPROVE_OPTION) { - String path = fileChooser.getSelectedFile().getPath(); - setContentPath(path); - } - - updateHelper(); - }//GEN-LAST:event_browseButtonActionPerformed - - // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JButton browseButton; - private javax.swing.JLabel errorLabel; - private javax.swing.JLabel pathLabel; - private javax.swing.JTextField pathTextField; - // End of variables declaration//GEN-END:variables - - /** - * Get the path of the user selected archive. - * - * @return the archive path - */ - public String getContentPaths() { - return pathTextField.getText(); - } - - /** - * Set the path of the archive file. - * - * @param s path of the archive file - */ - public void setContentPath(String s) { - pathTextField.setText(s); - } - - public void reset() { - //reset the UI elements to default - pathTextField.setText(null); - } - - /** - * Should we enable the next button of the wizard? - * - * @return true if a proper archive has been selected, false otherwise - */ - @NbBundle.Messages({"DataSourceOnCDriveError.text=Warning: Path to multi-user data source is on \"C:\" drive", - "DataSourceOnCDriveError.noOpenCase.errMsg=Warning: Exception while getting open case." - }) - public boolean validatePanel() { - errorLabel.setVisible(false); - String path = getContentPaths(); - if (StringUtils.isBlank(path)) { - return false; - } - - // display warning if there is one (but don't disable "next" button) - try { - if (false == PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) { - errorLabel.setVisible(true); - errorLabel.setText(Bundle.DataSourceOnCDriveError_text()); - } - } catch (NoCurrentCaseException ex) { - errorLabel.setVisible(true); - errorLabel.setText(Bundle.DataSourceOnCDriveError_noOpenCase_errMsg()); - } - - return new File(path).isFile() - || DriveUtils.isPhysicalDrive(path) - || DriveUtils.isPartition(path); - } - - public void storeSettings() { - String archivePathName = getContentPaths(); - if (null != archivePathName) { - String archivePath = archivePathName.substring(0, archivePathName.lastIndexOf(File.separator) + 1); - ModuleSettings.setConfigSetting(contextName, PROP_LAST_ARCHIVE_PATH, archivePath); - } - } - - public void readSettings() { - String lastArchivePath = ModuleSettings.getConfigSetting(contextName, PROP_LAST_ARCHIVE_PATH); - if (StringUtils.isNotBlank(lastArchivePath)) { - setContentPath(lastArchivePath); - } - } - - @Override - public void insertUpdate(DocumentEvent e) { - updateHelper(); - } - - @Override - public void removeUpdate(DocumentEvent e) { - updateHelper(); - } - - @Override - public void changedUpdate(DocumentEvent e) { - updateHelper(); - } - - /** - * Update functions are called by the pathTextField which has this set as - * it's DocumentEventListener. Each update function fires a property change - * to be caught by the parent panel. - * - */ - @NbBundle.Messages({"ArchiveFilePanel.moduleErr=Module Error", - "ArchiveFilePanel.moduleErr.msg=A module caused an error listening to ArchiveFilePanel updates." - + " See log to determine which module. Some data could be incomplete.\n"}) - private void updateHelper() { - try { - firePropertyChange(DataSourceProcessor.DSP_PANEL_EVENT.UPDATE_UI.toString(), false, true); - } catch (Exception e) { - logger.log(Level.SEVERE, "ArchiveFilePanel listener threw exception", e); //NON-NLS - MessageNotifyUtil.Notify.error(ArchiveFilePanel_moduleErr(), ArchiveFilePanel_moduleErr_msg()); - } - } - - /** - * Set the focus to the pathTextField. - */ - public void select() { - pathTextField.requestFocusInWindow(); - } -} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java deleted file mode 100644 index c22dfbd4ee..0000000000 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2015 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.experimental.autoingest; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.RandomAccessFile; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import javax.swing.filechooser.FileFilter; -import net.sf.sevenzipjbinding.ISequentialOutStream; -import net.sf.sevenzipjbinding.ISevenZipInArchive; -import net.sf.sevenzipjbinding.SevenZip; -import net.sf.sevenzipjbinding.SevenZipException; -import net.sf.sevenzipjbinding.SevenZipNativeInitializationException; -import net.sf.sevenzipjbinding.impl.RandomAccessFileInStream; -import net.sf.sevenzipjbinding.simple.ISimpleInArchive; -import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.GeneralFilter; -import org.sleuthkit.autopsy.coreutils.FileUtil; - -/** - * Set of utilities that handles archive file extraction. Uses 7zip library. - */ -final class ArchiveUtil { - - private static final String[] SUPPORTED_EXTENSIONS = {"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",}; // NON-NLS - private static final List ARCHIVE_EXTS = Arrays.asList(".zip", ".rar", ".arj", ".7z", ".7zip", ".gzip", ".gz", ".bzip2", ".tar", ".tgz"); //NON-NLS - @NbBundle.Messages("GeneralFilter.archiveDesc.text=Archive Files (.zip, .rar, .arj, .7z, .7zip, .gzip, .gz, .bzip2, .tar, .tgz)") - private static final String ARCHIVE_DESC = Bundle.GeneralFilter_archiveDesc_text(); - private static final GeneralFilter SEVEN_ZIP_FILTER = new GeneralFilter(ARCHIVE_EXTS, ARCHIVE_DESC); - private static final List ARCHIVE_FILTERS = new ArrayList<>(); - static { - ARCHIVE_FILTERS.add(SEVEN_ZIP_FILTER); - } - - private ArchiveUtil() { - } - - static List getArchiveFilters() { - return ARCHIVE_FILTERS; - } - - static boolean isArchive(Path dataSourcePath) { - String fileName = dataSourcePath.getFileName().toString(); - // check whether it's a zip archive file that can be extracted - return isAcceptedByFiler(new File(fileName), ARCHIVE_FILTERS); - } - - private static boolean isAcceptedByFiler(File file, List filters) { - for (FileFilter filter : filters) { - if (filter.accept(file)) { - return true; - } - } - return false; - } - - /** - * Enum of mime types which support archive extraction - */ - private enum SupportedArchiveExtractionFormats { - - ZIP("application/zip"), //NON-NLS - SEVENZ("application/x-7z-compressed"), //NON-NLS - GZIP("application/gzip"), //NON-NLS - XGZIP("application/x-gzip"), //NON-NLS - XBZIP2("application/x-bzip2"), //NON-NLS - XTAR("application/x-tar"), //NON-NLS - XGTAR("application/x-gtar"), - XRAR("application/x-rar-compressed"); //NON-NLS - - private final String mimeType; - - SupportedArchiveExtractionFormats(final String mimeType) { - this.mimeType = mimeType; - } - - @Override - public String toString() { - return this.mimeType; - } - } - - /** - * Exception thrown when archive handling resulted in an error - */ - static class ArchiveExtractionException extends Exception { - - private static final long serialVersionUID = 1L; - - ArchiveExtractionException(String message) { - super(message); - } - - ArchiveExtractionException(String message, Throwable cause) { - super(message, cause); - } - } - - /** - * This method returns array of supported file extensions. - * - * @return String array of supported file extensions. - */ - static String[] getSupportedArchiveTypes(){ - return SUPPORTED_EXTENSIONS; - } - - /** - * This method returns true if the MIME type is currently supported. Else it - * returns false. - * - * @param mimeType File mime type - * - * @return This method returns true if the file format is currently - * supported. Else it returns false. - */ - static boolean isExtractionSupportedByMimeType(String mimeType) { - for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) { - if (s.toString().equals(mimeType)) { - return true; - } - } - return false; - } - - /** - * This method returns true if the file extension is currently supported. - * Else it returns false. Attempt extension based detection in case Apache - * Tika based detection fails. - * - * @param extension File extension - * - * @return This method returns true if the file format is currently - * supported. Else it returns false. - */ - static boolean isExtractionSupportedByFileExtension(String extension) { - // attempt extension matching - for (String supportedExtension : SUPPORTED_EXTENSIONS) { - if (extension.equals(supportedExtension)) { - return true; - } - } - return false; - } - - /** - * Returns a list of file names contained within an archive. - * - * @param archiveFilePath Full path to the archive file - * - * @return List of file names contained within archive - * - * @throws - * ArchiveExtractionException - */ - static List getListOfFilesWithinArchive(String archiveFilePath) throws ArchiveExtractionException { - if (!SevenZip.isInitializedSuccessfully() && (SevenZip.getLastInitializationException() == null)) { - try { - SevenZip.initSevenZipFromPlatformJAR(); - } catch (SevenZipNativeInitializationException ex) { - throw new ArchiveExtractionException("AutoIngestDashboard_bnPause_paused", ex); - } - } - List files = new ArrayList<>(); - ISevenZipInArchive inArchive = null; - try { - RandomAccessFile randomAccessFile = new RandomAccessFile(new File(archiveFilePath), "r"); - inArchive = SevenZip.openInArchive(null, new RandomAccessFileInStream(randomAccessFile)); - final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface(); - for (ISimpleInArchiveItem item : simpleInArchive.getArchiveItems()) { - files.add(item.getPath()); - } - } catch (Exception ex) { - throw new ArchiveExtractionException("Exception while reading archive contents", ex); - } finally { - if (inArchive != null) { - try { - inArchive.close(); - } catch (SevenZipException ex) { - throw new ArchiveExtractionException("Exception while closing the archive", ex); - } - } - } - return files; - } - - /** - * Extracts contents of an archive file into a directory. - * - * @param archiveFilePath Full path to archive. - * @param destinationFolder Path to directory where results will be - * extracted to. - * - * @return List of file names contained within archive - * @throws - * ArchiveExtractionException - */ - static List unpackArchiveFile(String archiveFilePath, String destinationFolder) throws ArchiveExtractionException { - if (!SevenZip.isInitializedSuccessfully() && (SevenZip.getLastInitializationException() == null)) { - try { - SevenZip.initSevenZipFromPlatformJAR(); - } catch (SevenZipNativeInitializationException ex) { - throw new ArchiveExtractionException("Unable to initialize 7Zip libraries", ex); - } - } - List files = new ArrayList<>(); - ISevenZipInArchive inArchive = null; - try { - RandomAccessFile randomAccessFile = new RandomAccessFile(new File(archiveFilePath), "r"); - inArchive = SevenZip.openInArchive(null, new RandomAccessFileInStream(randomAccessFile)); - final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface(); - - for (ISimpleInArchiveItem entry : simpleInArchive.getArchiveItems()) { - String entryPathInArchive = entry.getPath(); - Path fullPath = Paths.get(destinationFolder, FileUtil.escapeFileName(entryPathInArchive)); // remove illegal characters from file name - File destFile = new File(fullPath.toString()); - File destinationParent = destFile.getParentFile(); - destinationParent.mkdirs(); - if (!entry.isFolder()) { - UnpackStream unpackStream = null; - try { - Long size = entry.getSize(); - unpackStream = new UnpackStream(destFile.toString(), size); - entry.extractSlow(unpackStream); - } catch (Exception ex) { - throw new ArchiveExtractionException("Exception while unpacking archive contents", ex); - } finally { - if (unpackStream != null) { - unpackStream.close(); - } - } - } - // keep track of extracted files - files.add(fullPath.toString()); - } - } catch (Exception ex) { - throw new ArchiveExtractionException("Exception while unpacking archive contents", ex); - } finally { - try { - if (inArchive != null) { - inArchive.close(); - } - } catch (SevenZipException ex) { - throw new ArchiveExtractionException("Exception while closing the archive", ex); - } - } - return files; - } - - /** - * Stream used to unpack an archive to local file - */ - private static class UnpackStream implements ISequentialOutStream { - - private OutputStream output; - private String destFilePath; - - UnpackStream(String destFilePath, long size) throws ArchiveExtractionException { - this.destFilePath = destFilePath; - try { - output = new FileOutputStream(destFilePath); - } catch (IOException ex) { - throw new ArchiveExtractionException("Exception while unpacking archive contents", ex); - } - - } - - @Override - public int write(byte[] bytes) throws SevenZipException { - try { - output.write(bytes); - } catch (IOException ex) { - throw new SevenZipException("Error writing unpacked file to " + destFilePath, ex); - } - return bytes.length; - } - - public void close() throws ArchiveExtractionException { - if (output != null) { - try { - output.flush(); - output.close(); - } catch (IOException ex) { - throw new ArchiveExtractionException("Exception while closing the archive", ex); - } - } - } - } -} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 55332cc03c..cf0e5c615d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -32,7 +32,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; -import java.sql.SQLException; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -70,7 +69,6 @@ import org.sleuthkit.autopsy.coordinationservice.CoordinationService.Lock; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.core.ServicesMonitor; import org.sleuthkit.autopsy.core.ServicesMonitor.ServicesMonitorException; -import org.sleuthkit.autopsy.core.UserPreferencesException; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult; import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; @@ -92,6 +90,9 @@ import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration; import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; +import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSource; +import org.sleuthkit.autopsy.datasourceprocessors.AddDataSourceCallback; +import org.sleuthkit.autopsy.datasourceprocessors.DataSourceProcessorUtility; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.AutoIngestJobException; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeControlEvent.ControlEventType; import org.sleuthkit.autopsy.ingest.IngestJob; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties index 4d2ade1b0a..0f074ca11c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties @@ -219,10 +219,6 @@ AutoIngestMetricsDialog.reportTextArea.text= AutoIngestMetricsDialog.metricsButton.text=Generate Metrics Report AutoIngestMetricsDialog.closeButton.text=Close AutoIngestMetricsDialog.datePicker.toolTipText=Choose a date -ArchiveFilePanel.pathLabel.text=Browse for an archive file: -ArchiveFilePanel.browseButton.text=Browse -ArchiveFilePanel.pathTextField.text= -ArchiveFilePanel.errorLabel.text=Error Label AutoIngestMetricsDialog.startingDataLabel.text=Starting Date: AutoIngestControlPanel.bnDeprioritizeCase.text=Deprioritize Case AutoIngestControlPanel.bnDeprioritizeJob.text=Deprioritize Job diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED index c10ac5581f..a7336e26a6 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED @@ -8,9 +8,6 @@ AinStatusNode.status.shuttingdown=Shutting Down AinStatusNode.status.startingup=Starting Up AinStatusNode.status.title=Status AinStatusNode.status.unknown=Unknown -ArchiveDSP.dsType.text=Archive file -ArchiveFilePanel.moduleErr=Module Error -ArchiveFilePanel.moduleErr.msg=A module caused an error listening to ArchiveFilePanel updates. See log to determine which module. Some data could be incomplete.\n AutoIngestAdminActions.cancelJobAction.title=Cancel Job AutoIngestAdminActions.cancelModuleAction.title=Cancel Module AutoIngestAdminActions.deleteCaseAction.error=Failed to delete case. @@ -170,12 +167,9 @@ CTL_AutoIngestDashboardOpenAction=Auto Ingest Dashboard CTL_AutoIngestDashboardTopComponent=Auto Ingest Jobs CTL_CasesDashboardAction=Multi-User Cases Dashboard CTL_CasesDashboardTopComponent=Cases -DataSourceOnCDriveError.noOpenCase.errMsg=Warning: Exception while getting open case. -DataSourceOnCDriveError.text=Warning: Path to multi-user data source is on "C:" drive DeleteCaseInputDirectoriesAction.menuItemText=Delete Input Directories DeleteCasesAction.menuItemText=Delete Case and Jobs DeleteCasesForReprocessingAction.menuItemText=Delete for Reprocessing -GeneralFilter.archiveDesc.text=Archive Files (.zip, .rar, .arj, .7z, .7zip, .gzip, .gz, .bzip2, .tar, .tgz) HINT_CasesDashboardTopComponent=This is an adminstrative dashboard for multi-user cases OpenAutoIngestLogAction.deletedLogErrorMsg=The case auto ingest log has been deleted. OpenAutoIngestLogAction.logOpenFailedErrorMsg=Failed to open case auto ingest log. See application log for details. @@ -377,10 +371,6 @@ AutoIngestMetricsDialog.reportTextArea.text= AutoIngestMetricsDialog.metricsButton.text=Generate Metrics Report AutoIngestMetricsDialog.closeButton.text=Close AutoIngestMetricsDialog.datePicker.toolTipText=Choose a date -ArchiveFilePanel.pathLabel.text=Browse for an archive file: -ArchiveFilePanel.browseButton.text=Browse -ArchiveFilePanel.pathTextField.text= -ArchiveFilePanel.errorLabel.text=Error Label AutoIngestMetricsDialog.startingDataLabel.text=Starting Date: AutoIngestControlPanel.bnDeprioritizeCase.text=Deprioritize Case AutoIngestControlPanel.bnDeprioritizeJob.text=Deprioritize Job