diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java index 506084f0c3..dd73b6f6ca 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java @@ -74,7 +74,7 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer private final Outline outline; private ExplorerManager explorerManager; - private NSDictionary rootDict; + private NSObject rootDict; /** * Creates new form PListViewer @@ -415,22 +415,35 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer } /** - * Parses given binary stream and extracts Plist key/value + * Parses given binary stream and extracts Plist key/value. * - * @param plistbytes + * @param plistbytes The byte array containing the Plist data. * * @return list of PropKeyValue */ private List parsePList(final byte[] plistbytes) throws IOException, PropertyListFormatException, ParseException, ParserConfigurationException, SAXException { final List plist = new ArrayList<>(); - rootDict = (NSDictionary) PropertyListParser.parse(plistbytes); + rootDict = PropertyListParser.parse(plistbytes); - final String[] keys = rootDict.allKeys(); - for (final String key : keys) { - final PropKeyValue pkv = parseProperty(key, rootDict.objectForKey(key)); - if (null != pkv) { - plist.add(pkv); + /* + * Parse the data if the root is an NSArray or NSDictionary. Anything + * else is unexpected and will be ignored. + */ + if (rootDict instanceof NSArray) { + for (int i=0; i < ((NSArray)rootDict).count(); i++) { + final PropKeyValue pkv = parseProperty("", ((NSArray)rootDict).objectAtIndex(i)); + if (null != pkv) { + plist.add(pkv); + } + } + } else if (rootDict instanceof NSDictionary) { + final String[] keys = ((NSDictionary)rootDict).allKeys(); + for (final String key : keys) { + final PropKeyValue pkv = parseProperty(key, ((NSDictionary)rootDict).objectForKey(key)); + if (null != pkv) { + plist.add(pkv); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 39e428d233..c1aec6c81b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -21,9 +21,6 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -93,7 +90,7 @@ class SevenZipExtractor { private String moduleDirAbsolute; private Blackboard blackboard; - + private ProgressHandle progress; private int numItems; private String currentArchiveName; @@ -164,18 +161,19 @@ class SevenZipExtractor { * * More heuristics to be added here * - * @param archiveFile the AbstractFile for the parent archive which - * which we are checking - * @param inArchive The SevenZip archive currently open for extraction - * - * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each - * file inside an archive is associated with a unique - * integer - * - * @param depthMap a concurrent hashmap which keeps track of the - * depth of all nested archives, key of objectID - * @param escapedFilePath the path to the archiveFileItem which has been - * escaped + * @param archiveFile the AbstractFile for the parent archive which + * which we are checking + * @param inArchive The SevenZip archive currently open for + * extraction + * + * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each + * file inside an archive is associated with a + * unique integer + * + * @param depthMap a concurrent hashmap which keeps track of the + * depth of all nested archives, key of objectID + * @param escapedFilePath the path to the archiveFileItem which has been + * escaped * * @return true if potential zip bomb, false otherwise */ @@ -551,7 +549,7 @@ class SevenZipExtractor { numItems = inArchive.getNumberOfItems(); progress.start(numItems); progressStarted = true; - + //setup the archive local root folder final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)); try { @@ -605,7 +603,7 @@ class SevenZipExtractor { inArchiveItemIndex, PropID.SIZE); if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && archiveItemSize != null && archiveItemSize > 0) { //if free space is known and file is not empty. String archiveItemPath = (String) inArchive.getProperty( - inArchiveItemIndex, PropID.PATH); + inArchiveItemIndex, PropID.PATH); long newDiskSpace = freeDiskSpace - archiveItemSize; if (newDiskSpace < MIN_FREE_DISK_SPACE) { String msg = NbBundle.getMessage(SevenZipExtractor.class, @@ -677,7 +675,7 @@ class SevenZipExtractor { inArchive.extract(extractionIndices, false, archiveCallBack); unpackSuccessful = unpackSuccessful & archiveCallBack.wasSuccessful(); - + archiveDetailsMap = null; // add them to the DB. We wait until the end so that we have the metadata on all of the @@ -795,140 +793,57 @@ class SevenZipExtractor { .mapToInt(Integer::intValue) .toArray(); } - + /** - * Stream used to unpack the archive to local file + * UnpackStream used by the SevenZipBindings to do archive extraction. A memory + * leak exists in the SevenZip library that will not let go of the streams until + * the entire archive extraction is complete. Instead of creating a new UnpackStream + * for every file in the archive, instead we just rebase our EncodedFileOutputStream pointer + * for every new file. */ - private abstract static class UnpackStream implements ISequentialOutStream { + private final static class UnpackStream implements ISequentialOutStream { - private OutputStream output; + private EncodedFileOutputStream output; private String localAbsPath; - - UnpackStream(String localAbsPath) { + private int bytesWritten; + + UnpackStream(String localAbsPath) throws IOException { + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; - try { - output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); //NON-NLS - } - + this.bytesWritten = 0; + } + + public void setNewOutputStream(String localAbsPath) throws IOException { + this.output.close(); + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; } - - public abstract long getSize(); - - OutputStream getOutput() { - return output; + + public int getSize() { + return bytesWritten; } - - String getLocalAbsPath() { - return localAbsPath; - } - - public void close() { - if (output != null) { - try { - output.flush(); - output.close(); - output = null; - } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS - } - } - } - } - - /** - * Stream used to unpack the archive of unknown size to local file - */ - private static class UnknownSizeUnpackStream extends UnpackStream { - - private long freeDiskSpace; - private boolean outOfSpace = false; - private long bytesWritten = 0; - - UnknownSizeUnpackStream(String localAbsPath, long freeDiskSpace) { - super(localAbsPath); - this.freeDiskSpace = freeDiskSpace; - } - - @Override - public long getSize() { - return this.bytesWritten; - } - + @Override public int write(byte[] bytes) throws SevenZipException { try { - // If the content size is unknown, cautiously write to disk. - // Write only if byte array is less than 80% of the current - // free disk space. - if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) { - getOutput().write(bytes); - // NOTE: this method is called multiple times for a - // single extractSlow() call. Update bytesWritten and - // freeDiskSpace after every write operation. - this.bytesWritten += bytes.length; - this.freeDiskSpace -= bytes.length; - } else { - this.outOfSpace = true; - logger.log(Level.INFO, NbBundle.getMessage( - SevenZipExtractor.class, - "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - } + output.write(bytes); + this.bytesWritten += bytes.length; } catch (IOException ex) { throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); + NbBundle.getMessage(SevenZipExtractor.class, + "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", + localAbsPath), ex); } return bytes.length; } - - @Override - public void close() { - if (getOutput() != null) { - try { - getOutput().flush(); - getOutput().close(); - if (this.outOfSpace) { - Files.delete(Paths.get(getLocalAbsPath())); - } - } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", getLocalAbsPath()); //NON-NLS - } - } - } - } - - /** - * Stream used to unpack the archive of known size to local file - */ - private static class KnownSizeUnpackStream extends UnpackStream { - - private long size; - - KnownSizeUnpackStream(String localAbsPath, long size) { - super(localAbsPath); - this.size = size; - } - - @Override - public long getSize() { - return this.size; - } - - @Override - public int write(byte[] bytes) throws SevenZipException { - try { - getOutput().write(bytes); - } catch (IOException ex) { - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); - } - return bytes.length; + + public void close() throws IOException { + try(EncodedFileOutputStream out = output) { + out.flush(); + } } + } /** @@ -973,9 +888,8 @@ class SevenZipExtractor { private UnpackStream unpackStream = null; private final Map archiveDetailsMap; private final ProgressHandle progressHandle; - + private int inArchiveItemIndex; - private final long freeDiskSpace; private long createTimeInSeconds; private long modTimeInSeconds; @@ -992,7 +906,6 @@ class SevenZipExtractor { String password, long freeDiskSpace) { this.inArchive = inArchive; - this.freeDiskSpace = freeDiskSpace; this.progressHandle = progressHandle; this.archiveFile = archiveFile; this.archiveDetailsMap = archiveDetailsMap; @@ -1000,19 +913,21 @@ class SevenZipExtractor { } /** - * Get stream is called by the internal framework as it traverses - * the archive structure. The ISequentialOutStream is where the - * archive file contents will be expanded and written to the local disk. - * + * Get stream is called by the internal framework as it traverses the + * archive structure. The ISequentialOutStream is where the archive file + * contents will be expanded and written to the local disk. + * * Skips folders, as there is nothing to extract. - * - * @param inArchiveItemIndex current location of the - * @param mode Will always be EXTRACT + * + * @param inArchiveItemIndex current location of the + * @param mode Will always be EXTRACT + * * @return - * @throws SevenZipException + * + * @throws SevenZipException */ @Override - public ISequentialOutStream getStream(int inArchiveItemIndex, + public ISequentialOutStream getStream(int inArchiveItemIndex, ExtractAskMode mode) throws SevenZipException { this.inArchiveItemIndex = inArchiveItemIndex; @@ -1023,28 +938,36 @@ class SevenZipExtractor { return null; } - final Long archiveItemSize = (Long) inArchive.getProperty( - inArchiveItemIndex, PropID.SIZE); final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); - - if (archiveItemSize != null) { - unpackStream = new SevenZipExtractor.KnownSizeUnpackStream( - localAbsPath, archiveItemSize); - } else { - unpackStream = new SevenZipExtractor.UnknownSizeUnpackStream( - localAbsPath, freeDiskSpace); + + //If the Unpackstream has been allocated, then set the Outputstream + //to another file rather than creating a new unpack stream. The 7Zip + //binding has a memory leak, so creating new unpack streams will not be + //dereferenced. As a fix, we create one UnpackStream, and mutate its state, + //so that there only exists one 8192 byte buffer in memory per archive. + try { + if (unpackStream != null) { + unpackStream.setNewOutputStream(localAbsPath); + } else { + unpackStream = new UnpackStream(localAbsPath); + } + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS + + "for archive file at %s", localAbsPath), ex.getMessage()); //NON-NLS + return null; } return unpackStream; } /** - * Retrieves the file metadata from the archive before extraction. + * Retrieves the file metadata from the archive before extraction. * Called after getStream. - * + * * @param mode Will always be EXTRACT. - * @throws SevenZipException + * + * @throws SevenZipException */ @Override public void prepareOperation(ExtractAskMode mode) throws SevenZipException { @@ -1061,18 +984,18 @@ class SevenZipExtractor { : writeTime.getTime() / 1000; accessTimeInSeconds = accessTime == null ? 0L : accessTime.getTime() / 1000; - + progressHandle.progress(archiveFile.getName() + ": " + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH), inArchiveItemIndex); - + } /** * Updates the unpackedNode data in the tree after the archive has been - * expanded to local disk. + * expanded to local disk. * - * @param result - ExtractOperationResult + * @param result - ExtractOperationResult * * @throws SevenZipException */ @@ -1089,7 +1012,7 @@ class SevenZipExtractor { localRelPath); return; } - + final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); if (result != ExtractOperationResult.OK) { @@ -1103,7 +1026,11 @@ class SevenZipExtractor { !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER), 0L, createTimeInSeconds, accessTimeInSeconds, modTimeInSeconds, localRelPath); - unpackStream.close(); + try { + unpackStream.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS + } } @Override @@ -1214,9 +1141,9 @@ class SevenZipExtractor { */ List getRootFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { ret.add(child.getFile()); - } + }); return ret; } @@ -1228,17 +1155,17 @@ class SevenZipExtractor { */ List getAllFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { getAllFileObjectsRec(ret, child); - } + }); return ret; } private void getAllFileObjectsRec(List list, UnpackedNode parent) { list.add(parent.getFile()); - for (UnpackedNode child : parent.getChildren()) { + parent.getChildren().forEach((child) -> { getAllFileObjectsRec(list, child); - } + }); } /** diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index 0d9210df27..13a5078658 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -272,7 +272,8 @@ class ReportHTML implements TableReportModule { in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/accounts.png"); //NON-NLS break; case TSK_WIFI_NETWORK: - in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + break; default: logger.log(Level.WARNING, "useDataTypeIcon: unhandled artifact type = {0}", dataType); //NON-NLS in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/star.png"); //NON-NLS diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 666a2cd02d..6881bb2e88 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeListener; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -68,6 +69,7 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; @@ -384,6 +386,51 @@ public final class ImageGalleryController { } + /** + * Returns a map of all data source object ids, along with + * their DB build status. + * + * This includes any data sources already in the table, + * and any data sources that might have been added to the + * case, but are not in the datasources table. + * + * @return map of data source object ids and their Db build status. + */ + public Map getAllDataSourcesDrawableDBStatus() { + + Map dataSourceStatusMap = new HashMap<>(); + + // no current case open to check + if ((null == getDatabase()) || (null == getSleuthKitCase())) { + return dataSourceStatusMap; + } + + try { + Map knownDataSourceIds = getDatabase().getDataSourceDbBuildStatus(); + + List dataSources = getSleuthKitCase().getDataSources(); + Set caseDataSourceIds = new HashSet<>(); + dataSources.stream().map(DataSource::getId).forEach(caseDataSourceIds::add); + + // collect all data sources already in the table + knownDataSourceIds.entrySet().stream().forEach((Map.Entry t) -> { + dataSourceStatusMap.put(t.getKey(), t.getValue()); + }); + + // collect any new data sources in the case. + caseDataSourceIds.forEach((Long id) -> { + if (!knownDataSourceIds.containsKey(id)) { + dataSourceStatusMap.put(id, DrawableDbBuildStatusEnum.UNKNOWN); + } + }); + + return dataSourceStatusMap; + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Image Gallery failed to get data source DB status.", ex); + return dataSourceStatusMap; + } + } + public boolean hasTooManyFiles(DataSource datasource) throws TskCoreException { String whereClause = (datasource == null) ? "1 = 1" @@ -392,6 +439,28 @@ public final class ImageGalleryController { return sleuthKitCase.countFilesWhere(whereClause) > FILE_LIMIT; } + + /** + * Checks if the given data source has any files with no mimetype + * + * @param datasource + * @return true if the datasource has any files with no mime type + * @throws TskCoreException + */ + public boolean hasFilesWithNoMimetype(Content datasource) throws TskCoreException { + + // There are some special files/attributes in the root folder, like $BadClus:$Bad and $Security:$SDS + // The IngestTasksScheduler does not push them down to the ingest modules, + // and hence they do not have any assigned mimetype + String whereClause = "data_source_obj_id = " + datasource.getId() + + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + + " AND ( mime_type IS NULL )" + + " AND ( meta_addr >= 32 ) " + + " AND ( parent_path <> '/' )" + + " AND ( name NOT like '$%:%' )"; + + return sleuthKitCase.countFilesWhere(whereClause) > 0; + } synchronized private void shutDownDBExecutor() { if (dbExecutor != null) { @@ -762,9 +831,13 @@ public final class ImageGalleryController { } } progressHandle.finish(); - if (taskCompletionStatus) { - taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.COMPLETE); - } + + DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = + (taskCompletionStatus) ? + DrawableDB.DrawableDbBuildStatusEnum.COMPLETE : + DrawableDB.DrawableDbBuildStatusEnum.DEFAULT; + taskDB.insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus); + updateMessage(""); updateProgress(-1.0); } @@ -831,36 +904,4 @@ public final class ImageGalleryController { } } - /** - * Copy files from a newly added data source into the DB. Get all "drawable" - * files, based on extension and mime-type. After ingest we use file type id - * module and if necessary jpeg/png signature matching to add/remove files - */ - @NbBundle.Messages({"PrePopulateDataSourceFiles.committingDb.status=committing image/video database"}) - static class PrePopulateDataSourceFiles extends BulkTransferTask { - - /** - * @param dataSourceObjId The object ID of the DataSource that is being - * pre-populated into the DrawableDB. - * @param controller The controller for this task. - */ - PrePopulateDataSourceFiles(long dataSourceObjId, ImageGalleryController controller) { - super(dataSourceObjId, controller); - } - - @Override - protected void cleanup(boolean success) { - } - - @Override - void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDBTransaction) { - taskDB.insertBasicFileData(DrawableFile.create(f, false, false), tr, caseDBTransaction); - } - - @Override - @NbBundle.Messages({"PrePopulateDataSourceFiles.prepopulatingDb.status=prepopulating image/video database",}) - ProgressHandle getInitialProgressHandle() { - return ProgressHandle.createHandle(Bundle.PrePopulateDataSourceFiles_prepopulatingDb_status(), this); - } - } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index c0336b76b1..530e3eb7d6 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -43,6 +43,8 @@ import org.sleuthkit.autopsy.ingest.IngestManager.IngestJobEvent; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.DATA_ADDED; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.FILE_DONE; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisStartedEvent; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -281,8 +283,8 @@ public class ImageGalleryModule { //For a data source added on the local node, prepopulate all file data to drawable database if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { Content newDataSource = (Content) evt.getNewValue(); - if (con.isListeningEnabled()) { - con.queueDBTask(new ImageGalleryController.PrePopulateDataSourceFiles(newDataSource.getId(), controller)); + if (con.isListeningEnabled()) { + controller.getDatabase().insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.DEFAULT); } } break; @@ -326,40 +328,68 @@ public class ImageGalleryModule { @Override public void propertyChange(PropertyChangeEvent evt) { IngestJobEvent eventType = IngestJobEvent.valueOf(evt.getPropertyName()); - if (eventType != IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED - || ((AutopsyEvent) evt).getSourceType() != AutopsyEvent.SourceType.REMOTE) { - return; - } - // A remote node added a new data source and just finished ingest on it. - //drawable db is stale, and if ImageGallery is open, ask user what to do - + try { - ImageGalleryController con = getController(); - con.setStale(true); - if (con.isListeningEnabled()) { - SwingUtilities.invokeLater(() -> { - if (ImageGalleryTopComponent.isImageGalleryOpen()) { - int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); - - switch (showAnswer) { - case JOptionPane.YES_OPTION: - con.rebuildDB(); - break; - case JOptionPane.NO_OPTION: - case JOptionPane.CANCEL_OPTION: - default: - break; //do nothing - } + ImageGalleryController controller = getController(); + + if (eventType == IngestJobEvent.DATA_SOURCE_ANALYSIS_STARTED) { + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + if (controller.isListeningEnabled()) { + DataSourceAnalysisStartedEvent dataSourceAnalysisStartedEvent = (DataSourceAnalysisStartedEvent) evt; + Content dataSource = dataSourceAnalysisStartedEvent.getDataSource(); + + controller.getDatabase().insertOrUpdateDataSource(dataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS); } - }); + } + } else if (eventType == IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED) { + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + if (controller.isListeningEnabled()) { + DataSourceAnalysisCompletedEvent dataSourceAnalysisCompletedEvent = (DataSourceAnalysisCompletedEvent) evt; + Content dataSource = dataSourceAnalysisCompletedEvent.getDataSource(); + + DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = + controller.hasFilesWithNoMimetype(dataSource) ? + DrawableDB.DrawableDbBuildStatusEnum.DEFAULT : + DrawableDB.DrawableDbBuildStatusEnum.COMPLETE; + + controller.getDatabase().insertOrUpdateDataSource(dataSource.getId(), datasourceDrawableDBStatus); + } + return; + } + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) { + // A remote node added a new data source and just finished ingest on it. + //drawable db is stale, and if ImageGallery is open, ask user what to do + controller.setStale(true); + if (controller.isListeningEnabled()) { + SwingUtilities.invokeLater(() -> { + if (ImageGalleryTopComponent.isImageGalleryOpen()) { + int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + + switch (showAnswer) { + case JOptionPane.YES_OPTION: + controller.rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + default: + break; //do nothing + } + } + }); + } + } } - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS + } + catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error getting ImageGalleryController.", ex); //NON-NLS + logger.log(Level.SEVERE, "Error getting ImageGalleryController.", ex); //NON-NLS } } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index e51f27cdef..927dc6b70e 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -18,13 +18,10 @@ */ package org.sleuthkit.autopsy.imagegallery; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.function.Consumer; import java.util.logging.Level; import java.util.stream.Collectors; import javafx.application.Platform; @@ -55,10 +52,8 @@ import javafx.stage.Modality; import javax.swing.SwingUtilities; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.apache.commons.lang3.ObjectUtils.notEqual; -import org.apache.commons.lang3.StringUtils; import org.openide.explorer.ExplorerManager; import org.openide.explorer.ExplorerUtils; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; @@ -232,8 +227,8 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl @SuppressWarnings(value = "unchecked") ComboBox> comboBox = (ComboBox>) datasourceDialog.getDialogPane().lookup(".combo-box"); //set custom cell renderer - comboBox.setCellFactory((ListView> param) -> new DataSourceCell(dataSourcesTooManyFiles)); - comboBox.setButtonCell(new DataSourceCell(dataSourcesTooManyFiles)); + comboBox.setCellFactory((ListView> param) -> new DataSourceCell(dataSourcesTooManyFiles, controller.getAllDataSourcesDrawableDBStatus())); + comboBox.setButtonCell(new DataSourceCell(dataSourcesTooManyFiles, controller.getAllDataSourcesDrawableDBStatus())); DataSource dataSource = datasourceDialog.showAndWait().orElse(Optional.empty()).orElse(null); try { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 81431b771e..5a921e6130 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -19,11 +19,10 @@ package org.sleuthkit.autopsy.imagegallery.actions; import com.google.common.util.concurrent.ListenableFuture; -import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import java.awt.Component; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import static java.util.concurrent.Executors.newSingleThreadExecutor; +import java.util.Map; import java.util.logging.Level; import javafx.application.Platform; import javafx.scene.control.Alert; @@ -32,7 +31,6 @@ import javafx.scene.control.CheckBox; import javafx.scene.control.Label; import javafx.scene.layout.VBox; import javafx.stage.Modality; -import javax.annotation.concurrent.ThreadSafe; import javax.swing.ImageIcon; import javax.swing.JButton; import javax.swing.JMenuItem; @@ -50,11 +48,12 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.Installer; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.ImageGalleryController; import org.sleuthkit.autopsy.imagegallery.ImageGalleryModule; import org.sleuthkit.autopsy.imagegallery.ImageGalleryPreferences; import org.sleuthkit.autopsy.imagegallery.ImageGalleryTopComponent; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum; import org.sleuthkit.autopsy.imagegallery.gui.GuiUtils; import org.sleuthkit.autopsy.imagegallery.utils.TaskUtils; import static org.sleuthkit.autopsy.imagegallery.utils.TaskUtils.addFXCallback; @@ -70,6 +69,8 @@ import org.sleuthkit.datamodel.TskCoreException; "OpenAction.stale.confDlg.msg=The image / video database may be out of date. " + "Do you want to update and listen for further ingest results?\n" + "Choosing 'yes' will update the database and enable listening to future ingests.", + "OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet.\n" + + "Please run FileType and EXIF ingest modules.", "OpenAction.stale.confDlg.title=Image Gallery"}) public final class OpenAction extends CallableSystemAction { @@ -184,17 +185,43 @@ public final class OpenAction extends CallableSystemAction { } private void checkDBStale(ImageGalleryController controller) { - //check if db is stale on throw away bg thread and then react back on jfx thread. - ListenableFuture staleFuture = TaskUtils.getExecutorForClass(OpenAction.class) - .submit(controller::isDataSourcesTableStale); - addFXCallback(staleFuture, - dbIsStale -> { + + ListenableFuture> dataSourceStatusMapFuture = TaskUtils.getExecutorForClass(OpenAction.class) + .submit(controller::getAllDataSourcesDrawableDBStatus); + + addFXCallback(dataSourceStatusMapFuture, + dataSourceStatusMap -> { + + boolean dbIsStale = false; + for (Map.Entry entry : dataSourceStatusMap.entrySet()) { + DrawableDbBuildStatusEnum status = entry.getValue(); + if (DrawableDbBuildStatusEnum.COMPLETE != status) { + dbIsStale = true; + } + } + //back on fx thread. if (false == dbIsStale) { //drawable db is not stale, just open it openTopComponent(); } else { - //drawable db is stale, ask what to do + + // If there is only one datasource and it's in DEFAULT State - + // ingest modules need to be run on the data source + if (dataSourceStatusMap.size()== 1) { + Map.Entry entry = dataSourceStatusMap.entrySet().iterator().next(); + if (entry.getValue() == DrawableDbBuildStatusEnum.DEFAULT ) { + Alert alert = new Alert(Alert.AlertType.WARNING, Bundle.OpenAction_notAnalyzedDlg_msg(), ButtonType.OK); + alert.setTitle(Bundle.OpenAction_stale_confDlg_title()); + alert.initModality(Modality.APPLICATION_MODAL); + + alert.showAndWait(); + return; + } + } + + //drawable db is stale, + //ask what to do Alert alert = new Alert(Alert.AlertType.WARNING, Bundle.OpenAction_stale_confDlg_msg(), ButtonType.YES, ButtonType.NO, ButtonType.CANCEL); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 8c27f6cce5..13eea6d654 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -174,11 +174,14 @@ public final class DrawableDB { /** * Enum to track Image gallery db rebuild status for a data source + * + * DO NOT add in the middle. */ public enum DrawableDbBuildStatusEnum { - UNKNOWN, /// no known status - IN_PROGRESS, /// drawable db rebuild has been started for the data source - COMPLETE; /// drawable db rebuild is complete for the data source + UNKNOWN, /// no known status + IN_PROGRESS, /// ingest or db rebuild is in progress + COMPLETE, /// All files in the data source have had file type detected + DEFAULT; /// Not all files in the data source have had file type detected } //////////////general database logic , mostly borrowed from sleuthkitcase diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java index 08543a42d2..7626c4fd77 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java @@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.imagegallery.gui; import java.util.Map; import java.util.Optional; import javafx.scene.control.ListCell; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum; import org.sleuthkit.datamodel.DataSource; /** @@ -29,9 +31,23 @@ import org.sleuthkit.datamodel.DataSource; public class DataSourceCell extends ListCell> { private final Map dataSourcesTooManyFiles; + private final Map dataSourcesDrawableDBStatus; - public DataSourceCell(Map dataSourcesViewable) { - this.dataSourcesTooManyFiles = dataSourcesViewable; + /** + * + * @param dataSourcesTooManyFiles: a map of too many files indicator for + * each data source. + * Data sources with too many files may substantially slow down + * the system and hence are disabled for selection. + * @param dataSourcesDrawableDBStatus a map of drawable DB status for + * each data sources. + * Data sources in DEFAULT state are not fully analyzed yet and are + * disabled for selection. + */ + public DataSourceCell(Map dataSourcesTooManyFiles, Map dataSourcesDrawableDBStatus) { + this.dataSourcesTooManyFiles = dataSourcesTooManyFiles; + this.dataSourcesDrawableDBStatus = dataSourcesDrawableDBStatus; + } @Override @@ -43,14 +59,28 @@ public class DataSourceCell extends ListCell> { DataSource dataSource = item.orElse(null); String text = (dataSource == null) ? "All" : dataSource.getName() + " (Id: " + dataSource.getId() + ")"; Boolean tooManyFilesInDataSource = dataSourcesTooManyFiles.getOrDefault(dataSource, false); + + DrawableDbBuildStatusEnum dataSourceDBStatus = (dataSource != null) ? + dataSourcesDrawableDBStatus.get(dataSource.getId()) : DrawableDbBuildStatusEnum.UNKNOWN; + + Boolean dataSourceNotAnalyzed = (dataSourceDBStatus == DrawableDbBuildStatusEnum.DEFAULT); if (tooManyFilesInDataSource) { text += " - Too many files"; + } + if (dataSourceNotAnalyzed) { + text += " - Not Analyzed"; + } + + // check if item should be disabled + if (tooManyFilesInDataSource || dataSourceNotAnalyzed) { + setDisable(true); setStyle("-fx-opacity : .5"); - } else { + } + else { setGraphic(null); setStyle("-fx-opacity : 1"); } - setDisable(tooManyFilesInDataSource); + setText(text); } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java index bc0f5e5d84..4bcecbbecb 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java @@ -240,8 +240,8 @@ public class Toolbar extends ToolBar { } private void initDataSourceComboBox() { - dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable)); - dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable)); + dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus())); + dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus())); dataSourceComboBox.setConverter(new StringConverter>() { @Override public String toString(Optional object) { diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index 91571da9bf..202947f033 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Sat, 13 Oct 2018 21:02:18 -0400 +#Tue, 13 Nov 2018 17:30:09 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 @@ -8,4 +8,4 @@ SplashRunningTextBounds=0,289,538,18 SplashRunningTextColor=0x0 SplashRunningTextFontSize=19 -currentVersion=Autopsy 4.9.0 +currentVersion=Autopsy 4.9.1 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index 90fb6cf276..11be888847 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,4 +1,4 @@ #Updated by build script -#Sat, 13 Oct 2018 21:02:18 -0400 -CTL_MainWindow_Title=Autopsy 4.9.0 -CTL_MainWindow_Title_No_Project=Autopsy 4.9.0 +#Tue, 13 Nov 2018 17:30:09 -0500 +CTL_MainWindow_Title=Autopsy 4.9.1 +CTL_MainWindow_Title_No_Project=Autopsy 4.9.1