From c9913f24e8b54119aa38335e6eccb8f40e60b147 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 9 Oct 2018 15:05:42 -0400 Subject: [PATCH 01/14] Fixed the source of the memory issue and did some light refactoring --- .../SevenZipExtractor.java | 224 ++++++------------ 1 file changed, 74 insertions(+), 150 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 704bdac4f8..df208b14c6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -46,6 +47,7 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -93,7 +95,7 @@ class SevenZipExtractor { private String moduleDirAbsolute; private Blackboard blackboard; - + private ProgressHandle progress; private int numItems; private String currentArchiveName; @@ -164,18 +166,19 @@ class SevenZipExtractor { * * More heuristics to be added here * - * @param archiveFile the AbstractFile for the parent archive which - * which we are checking - * @param inArchive The SevenZip archive currently open for extraction - * - * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each - * file inside an archive is associated with a unique - * integer - * - * @param depthMap a concurrent hashmap which keeps track of the - * depth of all nested archives, key of objectID - * @param escapedFilePath the path to the archiveFileItem which has been - * escaped + * @param archiveFile the AbstractFile for the parent archive which + * which we are checking + * @param inArchive The SevenZip archive currently open for + * extraction + * + * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each + * file inside an archive is associated with a + * unique integer + * + * @param depthMap a concurrent hashmap which keeps track of the + * depth of all nested archives, key of objectID + * @param escapedFilePath the path to the archiveFileItem which has been + * escaped * * @return true if potential zip bomb, false otherwise */ @@ -543,7 +546,7 @@ class SevenZipExtractor { logger.log(Level.INFO, "Count of items in archive: {0}: {1}", new Object[]{escapedArchiveFilePath, numItems}); //NON-NLS progress.start(numItems); progressStarted = true; - + //setup the archive local root folder final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)); try { @@ -597,7 +600,7 @@ class SevenZipExtractor { inArchiveItemIndex, PropID.SIZE); if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && archiveItemSize != null && archiveItemSize > 0) { //if free space is known and file is not empty. String archiveItemPath = (String) inArchive.getProperty( - inArchiveItemIndex, PropID.PATH); + inArchiveItemIndex, PropID.PATH); long newDiskSpace = freeDiskSpace - archiveItemSize; if (newDiskSpace < MIN_FREE_DISK_SPACE) { String msg = NbBundle.getMessage(SevenZipExtractor.class, @@ -669,7 +672,7 @@ class SevenZipExtractor { inArchive.extract(extractionIndices, false, archiveCallBack); unpackSuccessful = unpackSuccessful & archiveCallBack.wasSuccessful(); - + archiveDetailsMap = null; // add them to the DB. We wait until the end so that we have the metadata on all of the @@ -788,62 +791,22 @@ class SevenZipExtractor { .toArray(); } - /** - * Stream used to unpack the archive to local file - */ - private abstract static class UnpackStream implements ISequentialOutStream { - - private OutputStream output; - private String localAbsPath; - - UnpackStream(String localAbsPath) { - this.localAbsPath = localAbsPath; - try { - output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); //NON-NLS - } - - } - - public abstract long getSize(); - - OutputStream getOutput() { - return output; - } - - String getLocalAbsPath() { - return localAbsPath; - } - - public void close() { - if (output != null) { - try { - output.flush(); - output.close(); - output = null; - } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS - } - } - } - } - /** * Stream used to unpack the archive of unknown size to local file */ - private static class UnknownSizeUnpackStream extends UnpackStream { + private static class UnpackStream implements ISequentialOutStream, AutoCloseable { - private long freeDiskSpace; - private boolean outOfSpace = false; - private long bytesWritten = 0; + private EncodedFileOutputStream output; + private String localAbsPath; - UnknownSizeUnpackStream(String localAbsPath, long freeDiskSpace) { - super(localAbsPath); - this.freeDiskSpace = freeDiskSpace; + private long bytesWritten; + + UnpackStream(String localAbsPath) throws IOException { + output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; } - @Override public long getSize() { return this.bytesWritten; } @@ -851,78 +814,36 @@ class SevenZipExtractor { @Override public int write(byte[] bytes) throws SevenZipException { try { - // If the content size is unknown, cautiously write to disk. - // Write only if byte array is less than 80% of the current - // free disk space. - if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) { - getOutput().write(bytes); - // NOTE: this method is called multiple times for a - // single extractSlow() call. Update bytesWritten and - // freeDiskSpace after every write operation. - this.bytesWritten += bytes.length; - this.freeDiskSpace -= bytes.length; - } else { - this.outOfSpace = true; - logger.log(Level.INFO, NbBundle.getMessage( - SevenZipExtractor.class, - "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - } + output.write(bytes); + // Update bytesWritten + this.bytesWritten += bytes.length; } catch (IOException ex) { throw new SevenZipException( NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); + localAbsPath), ex); } return bytes.length; } + public void setNewOutputStream(String localAbsPath) throws IOException { + this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; + } + @Override public void close() { - if (getOutput() != null) { + if (output != null) { try { - getOutput().flush(); - getOutput().close(); - if (this.outOfSpace) { - Files.delete(Paths.get(getLocalAbsPath())); - } + output.flush(); + output.close(); } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", getLocalAbsPath()); //NON-NLS + logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS } } } } - /** - * Stream used to unpack the archive of known size to local file - */ - private static class KnownSizeUnpackStream extends UnpackStream { - - private long size; - - KnownSizeUnpackStream(String localAbsPath, long size) { - super(localAbsPath); - this.size = size; - } - - @Override - public long getSize() { - return this.size; - } - - @Override - public int write(byte[] bytes) throws SevenZipException { - try { - getOutput().write(bytes); - } catch (IOException ex) { - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); - } - return bytes.length; - } - } - /** * Wrapper for necessary details used in StandardIArchiveExtractCallback */ @@ -965,9 +886,8 @@ class SevenZipExtractor { private UnpackStream unpackStream = null; private final Map archiveDetailsMap; private final ProgressHandle progressHandle; - + private int inArchiveItemIndex; - private final long freeDiskSpace; private long createTimeInSeconds; private long modTimeInSeconds; @@ -984,7 +904,6 @@ class SevenZipExtractor { String password, long freeDiskSpace) { this.inArchive = inArchive; - this.freeDiskSpace = freeDiskSpace; this.progressHandle = progressHandle; this.archiveFile = archiveFile; this.archiveDetailsMap = archiveDetailsMap; @@ -992,19 +911,21 @@ class SevenZipExtractor { } /** - * Get stream is called by the internal framework as it traverses - * the archive structure. The ISequentialOutStream is where the - * archive file contents will be expanded and written to the local disk. - * + * Get stream is called by the internal framework as it traverses the + * archive structure. The ISequentialOutStream is where the archive file + * contents will be expanded and written to the local disk. + * * Skips folders, as there is nothing to extract. - * - * @param inArchiveItemIndex current location of the - * @param mode Will always be EXTRACT + * + * @param inArchiveItemIndex current location of the + * @param mode Will always be EXTRACT + * * @return - * @throws SevenZipException + * + * @throws SevenZipException */ @Override - public ISequentialOutStream getStream(int inArchiveItemIndex, + public ISequentialOutStream getStream(int inArchiveItemIndex, ExtractAskMode mode) throws SevenZipException { this.inArchiveItemIndex = inArchiveItemIndex; @@ -1015,28 +936,31 @@ class SevenZipExtractor { return null; } - final Long archiveItemSize = (Long) inArchive.getProperty( - inArchiveItemIndex, PropID.SIZE); final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); - - if (archiveItemSize != null) { - unpackStream = new SevenZipExtractor.KnownSizeUnpackStream( - localAbsPath, archiveItemSize); - } else { - unpackStream = new SevenZipExtractor.UnknownSizeUnpackStream( - localAbsPath, freeDiskSpace); + + try { + if (unpackStream != null) { + unpackStream.setNewOutputStream(localAbsPath); + } else { + unpackStream = new UnpackStream(localAbsPath); + } + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS + + "for archive file at %s", localAbsPath), ex); //NON-NLS + return null; } return unpackStream; } /** - * Retrieves the file metadata from the archive before extraction. + * Retrieves the file metadata from the archive before extraction. * Called after getStream. - * + * * @param mode Will always be EXTRACT. - * @throws SevenZipException + * + * @throws SevenZipException */ @Override public void prepareOperation(ExtractAskMode mode) throws SevenZipException { @@ -1053,18 +977,18 @@ class SevenZipExtractor { : writeTime.getTime() / 1000; accessTimeInSeconds = accessTime == null ? 0L : accessTime.getTime() / 1000; - + progressHandle.progress(archiveFile.getName() + ": " + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH), inArchiveItemIndex); - + } /** * Updates the unpackedNode data in the tree after the archive has been - * expanded to local disk. + * expanded to local disk. * - * @param result - ExtractOperationResult + * @param result - ExtractOperationResult * * @throws SevenZipException */ @@ -1081,7 +1005,7 @@ class SevenZipExtractor { localRelPath); return; } - + final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); if (result != ExtractOperationResult.OK) { From c88ea15bbcedfd340a23ffb5672295a92f18cba0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 9 Oct 2018 16:22:50 -0400 Subject: [PATCH 02/14] Added some comments and did some testing, everything looks good --- .../SevenZipExtractor.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index df208b14c6..977340600b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -19,12 +19,8 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -47,7 +43,6 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -792,7 +787,8 @@ class SevenZipExtractor { } /** - * Stream used to unpack the archive of unknown size to local file + * Stream used to unpack the archive item to local file. This will be used when + * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. */ private static class UnpackStream implements ISequentialOutStream, AutoCloseable { @@ -815,7 +811,6 @@ class SevenZipExtractor { public int write(byte[] bytes) throws SevenZipException { try { output.write(bytes); - // Update bytesWritten this.bytesWritten += bytes.length; } catch (IOException ex) { throw new SevenZipException( @@ -825,6 +820,15 @@ class SevenZipExtractor { return bytes.length; } + /** + * Update the OutputStream to point to a new File. This method mutates the state so + * that there is no overhead of creating a new object and buffer. Additionally, the + * 7zip binding has a memory leak, so this prevents multiple streams from being created + * and never GC'd since the 7zip lib never frees a reference to them. + * + * @param localAbsPath Path to local file + * @throws IOException + */ public void setNewOutputStream(String localAbsPath) throws IOException { this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; @@ -939,6 +943,11 @@ class SevenZipExtractor { final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); + //If the Unpackstream has been allocated, then set the Outputstream + //to another file rather than creating a new unpack stream. The 7Zip + //binding has a memory leak, so creating new unpack streams will not be + //dereferenced. As a fix, we create one UnpackStream, and mutate its state, + //so that there only exists one 8192 byte buffer in memory per archive. try { if (unpackStream != null) { unpackStream.setNewOutputStream(localAbsPath); From 611353d8f1132755f00e925757cc38451be1061a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 11 Oct 2018 16:36:13 -0400 Subject: [PATCH 03/14] Created a solution entirely within the EFE module and not changing any public API in datamodel --- .../SevenZipExtractor.java | 123 ++++++++++++------ 1 file changed, 80 insertions(+), 43 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 977340600b..c8f0fce062 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -43,6 +43,7 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -790,34 +791,19 @@ class SevenZipExtractor { * Stream used to unpack the archive item to local file. This will be used when * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. */ - private static class UnpackStream implements ISequentialOutStream, AutoCloseable { + private final static class MutableEncodedFileOutputStream extends EncodedFileOutputStream + implements AutoCloseable { + + private static final int HEADER_LENGTH = 32; + private static final String HEADER = "TSK_CONTAINER_XOR1_xxxxxxxxxxxxx"; - private EncodedFileOutputStream output; - private String localAbsPath; - - private long bytesWritten; - - UnpackStream(String localAbsPath) throws IOException { - output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - this.localAbsPath = localAbsPath; - this.bytesWritten = 0; - } - - public long getSize() { - return this.bytesWritten; + MutableEncodedFileOutputStream(String localAbsPath) throws IOException { + super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); } @Override - public int write(byte[] bytes) throws SevenZipException { - try { - output.write(bytes); - this.bytesWritten += bytes.length; - } catch (IOException ex) { - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - localAbsPath), ex); - } - return bytes.length; + public void write(byte[] bytes) throws IOException { + super.write(bytes); } /** @@ -830,22 +816,69 @@ class SevenZipExtractor { * @throws IOException */ public void setNewOutputStream(String localAbsPath) throws IOException { - this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.out.close(); + this.out = new FileOutputStream(localAbsPath); + writeHeader(); + } + + private byte[] getEncodedHeader() { + byte [] encHeader = new byte[HEADER_LENGTH]; + byte [] plainHeader = HEADER.getBytes(); + + for(int i = 0; i < HEADER_LENGTH; i++){ + encHeader[i] = ((byte)(plainHeader[i] ^ 0xca)); + } + return encHeader; + } + + private void writeHeader() throws IOException { + // We get the encoded header here so it will be in plaintext after encoding + write(getEncodedHeader(), 0, HEADER_LENGTH); + } + } + + private final static class UnpackStream implements ISequentialOutStream { + + private final MutableEncodedFileOutputStream output; + private String localAbsPath; + private int bytesWritten; + + UnpackStream(String localAbsPath) throws IOException { + this.output = new MutableEncodedFileOutputStream(localAbsPath); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; + } + + public void setNewOutputStream(String localAbsPath) throws IOException { + this.output.setNewOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } - - @Override - public void close() { - if (output != null) { - try { - output.flush(); - output.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS - } - } + + public int getSize() { + return bytesWritten; } + + @Override + public int write(byte[] bytes) throws SevenZipException { + try { + output.write(bytes); + this.bytesWritten += bytes.length; + } catch (IOException ex) { + throw new SevenZipException( + NbBundle.getMessage(SevenZipExtractor.class, + "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", + localAbsPath), ex); + } + return bytes.length; + } + + public void close() throws IOException { + try(MutableEncodedFileOutputStream out = output) { + out.flush(); + } + } + } /** @@ -1028,7 +1061,11 @@ class SevenZipExtractor { !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER), 0L, createTimeInSeconds, accessTimeInSeconds, modTimeInSeconds, localRelPath); - unpackStream.close(); + try { + unpackStream.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS + } } @Override @@ -1139,9 +1176,9 @@ class SevenZipExtractor { */ List getRootFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { ret.add(child.getFile()); - } + }); return ret; } @@ -1153,17 +1190,17 @@ class SevenZipExtractor { */ List getAllFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { getAllFileObjectsRec(ret, child); - } + }); return ret; } private void getAllFileObjectsRec(List list, UnpackedNode parent) { list.add(parent.getFile()); - for (UnpackedNode child : parent.getChildren()) { + parent.getChildren().forEach((child) -> { getAllFileObjectsRec(list, child); - } + }); } /** From 6ae37630a432f4bb31c8f122294182053f1e3418 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 29 Oct 2018 15:41:59 -0400 Subject: [PATCH 04/14] Fixed codacy suggestions --- .../embeddedfileextractor/SevenZipExtractor.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index c8f0fce062..dc4f190c09 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -800,12 +800,7 @@ class SevenZipExtractor { MutableEncodedFileOutputStream(String localAbsPath) throws IOException { super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); } - - @Override - public void write(byte[] bytes) throws IOException { - super.write(bytes); - } - + /** * Update the OutputStream to point to a new File. This method mutates the state so * that there is no overhead of creating a new object and buffer. Additionally, the @@ -837,6 +832,14 @@ class SevenZipExtractor { } } + /** + * UnpackStream used by the SevenZipBindings to do archive extraction. A memory + * leak exists in the SevenZip library that will not let go of the Streams until + * the entire archive extraction is complete. So, to compensate this UnpackStream + * uses a MutableEncodedFileOutputStream, which supports setting a new disk + * location for the contents to be written to (as opposed to creating a new steam + * for each file). + */ private final static class UnpackStream implements ISequentialOutStream { private final MutableEncodedFileOutputStream output; From e4142387929817e2ad47810ea2e047a1ca333045 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 30 Oct 2018 08:39:03 -0400 Subject: [PATCH 05/14] This import keeps getting automatically added and its causing Codacy to complain --- .../autopsy/modules/embeddedfileextractor/SevenZipExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index dc4f190c09..0deea2baf0 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -43,7 +43,6 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; From 559a963a0bbbc51aa74984dce249d03ca13facf6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 5 Nov 2018 02:41:50 -0500 Subject: [PATCH 06/14] Added handling of NSArray data. --- .../autopsy/contentviewers/PListViewer.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java index 506084f0c3..dd73b6f6ca 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java @@ -74,7 +74,7 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer private final Outline outline; private ExplorerManager explorerManager; - private NSDictionary rootDict; + private NSObject rootDict; /** * Creates new form PListViewer @@ -415,22 +415,35 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer } /** - * Parses given binary stream and extracts Plist key/value + * Parses given binary stream and extracts Plist key/value. * - * @param plistbytes + * @param plistbytes The byte array containing the Plist data. * * @return list of PropKeyValue */ private List parsePList(final byte[] plistbytes) throws IOException, PropertyListFormatException, ParseException, ParserConfigurationException, SAXException { final List plist = new ArrayList<>(); - rootDict = (NSDictionary) PropertyListParser.parse(plistbytes); + rootDict = PropertyListParser.parse(plistbytes); - final String[] keys = rootDict.allKeys(); - for (final String key : keys) { - final PropKeyValue pkv = parseProperty(key, rootDict.objectForKey(key)); - if (null != pkv) { - plist.add(pkv); + /* + * Parse the data if the root is an NSArray or NSDictionary. Anything + * else is unexpected and will be ignored. + */ + if (rootDict instanceof NSArray) { + for (int i=0; i < ((NSArray)rootDict).count(); i++) { + final PropKeyValue pkv = parseProperty("", ((NSArray)rootDict).objectAtIndex(i)); + if (null != pkv) { + plist.add(pkv); + } + } + } else if (rootDict instanceof NSDictionary) { + final String[] keys = ((NSDictionary)rootDict).allKeys(); + for (final String key : keys) { + final PropKeyValue pkv = parseProperty(key, ((NSDictionary)rootDict).objectForKey(key)); + if (null != pkv) { + plist.add(pkv); + } } } From 3f2a748d9d48868c9774ad3ba328e72136710487 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:43:32 -0500 Subject: [PATCH 07/14] Fixing unncessary class indirection --- .../SevenZipExtractor.java | 53 ++----------------- 1 file changed, 4 insertions(+), 49 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 0deea2baf0..d3cc8f159f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -785,51 +785,6 @@ class SevenZipExtractor { .mapToInt(Integer::intValue) .toArray(); } - - /** - * Stream used to unpack the archive item to local file. This will be used when - * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. - */ - private final static class MutableEncodedFileOutputStream extends EncodedFileOutputStream - implements AutoCloseable { - - private static final int HEADER_LENGTH = 32; - private static final String HEADER = "TSK_CONTAINER_XOR1_xxxxxxxxxxxxx"; - - MutableEncodedFileOutputStream(String localAbsPath) throws IOException { - super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - } - - /** - * Update the OutputStream to point to a new File. This method mutates the state so - * that there is no overhead of creating a new object and buffer. Additionally, the - * 7zip binding has a memory leak, so this prevents multiple streams from being created - * and never GC'd since the 7zip lib never frees a reference to them. - * - * @param localAbsPath Path to local file - * @throws IOException - */ - public void setNewOutputStream(String localAbsPath) throws IOException { - this.out.close(); - this.out = new FileOutputStream(localAbsPath); - writeHeader(); - } - - private byte[] getEncodedHeader() { - byte [] encHeader = new byte[HEADER_LENGTH]; - byte [] plainHeader = HEADER.getBytes(); - - for(int i = 0; i < HEADER_LENGTH; i++){ - encHeader[i] = ((byte)(plainHeader[i] ^ 0xca)); - } - return encHeader; - } - - private void writeHeader() throws IOException { - // We get the encoded header here so it will be in plaintext after encoding - write(getEncodedHeader(), 0, HEADER_LENGTH); - } - } /** * UnpackStream used by the SevenZipBindings to do archive extraction. A memory @@ -841,18 +796,18 @@ class SevenZipExtractor { */ private final static class UnpackStream implements ISequentialOutStream { - private final MutableEncodedFileOutputStream output; + private EncodedFileOutputStream output; private String localAbsPath; private int bytesWritten; UnpackStream(String localAbsPath) throws IOException { - this.output = new MutableEncodedFileOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } public void setNewOutputStream(String localAbsPath) throws IOException { - this.output.setNewOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } @@ -876,7 +831,7 @@ class SevenZipExtractor { } public void close() throws IOException { - try(MutableEncodedFileOutputStream out = output) { + try(EncodedFileOutputStream out = output) { out.flush(); } } From 3040976ab81645ab670f0e49110a4c75d9fdf2c2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:55:11 -0500 Subject: [PATCH 08/14] Made only unpackstream wrap an encodedfileoutputstream --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index dba3e82e81..c9cdf59d74 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -809,13 +809,14 @@ class SevenZipExtractor { private int bytesWritten; UnpackStream(String localAbsPath) throws IOException { - this.output = new EncodedFileOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } public void setNewOutputStream(String localAbsPath) throws IOException { - this.output = new EncodedFileOutputStream(localAbsPath); + this.output.close(); + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } From e77faa4d2d011822c66a1951c760a775b1b9cc37 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:57:16 -0500 Subject: [PATCH 09/14] Fixed a log message --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index c9cdf59d74..45963d7cdd 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -955,7 +955,7 @@ class SevenZipExtractor { } } catch (IOException ex) { logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS - + "for archive file at %s", localAbsPath), ex); //NON-NLS + + "for archive file at %s", localAbsPath), ex.getMessage()); //NON-NLS return null; } From 061a5debb1edbabdcc217c0d60e1ede7f09a0b15 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 9 Nov 2018 17:46:55 -0500 Subject: [PATCH 10/14] Add missing break to case statement for HTML Report generation --- Core/src/org/sleuthkit/autopsy/report/ReportHTML.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index 0d9210df27..13a5078658 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -272,7 +272,8 @@ class ReportHTML implements TableReportModule { in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/accounts.png"); //NON-NLS break; case TSK_WIFI_NETWORK: - in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + break; default: logger.log(Level.WARNING, "useDataTypeIcon: unhandled artifact type = {0}", dataType); //NON-NLS in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/star.png"); //NON-NLS From ac54dee5a2d843cc24088016b009a3625d1ccae3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 12 Nov 2018 13:09:29 -0500 Subject: [PATCH 11/14] Updated method header --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 45963d7cdd..c1aec6c81b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -796,11 +796,10 @@ class SevenZipExtractor { /** * UnpackStream used by the SevenZipBindings to do archive extraction. A memory - * leak exists in the SevenZip library that will not let go of the Streams until - * the entire archive extraction is complete. So, to compensate this UnpackStream - * uses a MutableEncodedFileOutputStream, which supports setting a new disk - * location for the contents to be written to (as opposed to creating a new steam - * for each file). + * leak exists in the SevenZip library that will not let go of the streams until + * the entire archive extraction is complete. Instead of creating a new UnpackStream + * for every file in the archive, instead we just rebase our EncodedFileOutputStream pointer + * for every new file. */ private final static class UnpackStream implements ISequentialOutStream { From a1d0519b6ab1d61dfd14f93528c01176a5da9ac3 Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 13 Nov 2018 17:33:47 -0500 Subject: [PATCH 12/14] 1110: Remove pre-populating phase. --- .../imagegallery/ImageGalleryController.java | 111 ++++++++++++------ .../imagegallery/ImageGalleryModule.java | 91 +++++++++----- .../ImageGalleryTopComponent.java | 10 +- .../imagegallery/actions/OpenAction.java | 46 ++++++-- .../imagegallery/datamodel/DrawableDB.java | 9 +- .../imagegallery/gui/DataSourceCell.java | 25 +++- .../autopsy/imagegallery/gui/Toolbar.java | 4 +- 7 files changed, 205 insertions(+), 91 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 646affbad7..a0300e72c5 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeListener; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -67,8 +68,8 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.HashSetManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; @@ -382,6 +383,51 @@ public final class ImageGalleryController { } + /** + * Returns a map of all data source object ids, along with + * their DB build status. + * + * This includes any data sources already in the table, + * and any data sources that might have been added to the + * case, but are not in the datasources table. + * + * @return map of data source object ids and their Db build status. + */ + public Map getAllDataSourcesDrawableDBStatus() { + + Map dataSourceStatusMap = new HashMap<>(); + + // no current case open to check + if ((null == getDatabase()) || (null == getSleuthKitCase())) { + return dataSourceStatusMap; + } + + try { + Map knownDataSourceIds = getDatabase().getDataSourceDbBuildStatus(); + + List dataSources = getSleuthKitCase().getDataSources(); + Set caseDataSourceIds = new HashSet<>(); + dataSources.stream().map(DataSource::getId).forEach(caseDataSourceIds::add); + + // collect all data sources already in the table + knownDataSourceIds.entrySet().stream().forEach((Map.Entry t) -> { + dataSourceStatusMap.put(t.getKey(), t.getValue()); + }); + + // collect any new data sources in the case. + caseDataSourceIds.forEach((Long id) -> { + if (!knownDataSourceIds.containsKey(id)) { + dataSourceStatusMap.put(id, DrawableDbBuildStatusEnum.UNKNOWN); + } + }); + + return dataSourceStatusMap; + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Image Gallery failed to get data source DB status.", ex); + return dataSourceStatusMap; + } + } + public boolean hasTooManyFiles(DataSource datasource) throws TskCoreException { String whereClause = (datasource == null) ? "1 = 1" @@ -390,6 +436,27 @@ public final class ImageGalleryController { return sleuthKitCase.countFilesWhere(whereClause) > FILE_LIMIT; } + + /** + * Checks if the given data source has any files with no mimetype + * + * @param datasource + * @return true if the datasource has any files with no mime type + * @throws TskCoreException + */ + public boolean hasFilesWithNoMimetype(Content datasource) throws TskCoreException { + + // There are some special files/attributes in the root folder, like $BadClus:$Bad and $Security:$SDS + // that do not go through any ingest modules, and hence do not have any assigned mimetype + String whereClause = "data_source_obj_id = " + datasource.getId() + + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + + " AND ( mime_type IS NULL )" + + " AND ( meta_addr >= 32 ) " + + " AND ( parent_path <> '/' )" + + " AND ( name NOT like '$%:%' )"; + + return sleuthKitCase.countFilesWhere(whereClause) > 0; + } synchronized private void shutDownDBExecutor() { if (dbExecutor != null) { @@ -763,9 +830,13 @@ public final class ImageGalleryController { } } progressHandle.finish(); - if (taskCompletionStatus) { - taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.COMPLETE); - } + + DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = + (taskCompletionStatus) ? + DrawableDB.DrawableDbBuildStatusEnum.COMPLETE : + DrawableDB.DrawableDbBuildStatusEnum.DEFAULT; + taskDB.insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus); + updateMessage(""); updateProgress(-1.0); } @@ -832,36 +903,4 @@ public final class ImageGalleryController { } } - /** - * Copy files from a newly added data source into the DB. Get all "drawable" - * files, based on extension and mime-type. After ingest we use file type id - * module and if necessary jpeg/png signature matching to add/remove files - */ - @NbBundle.Messages({"PrePopulateDataSourceFiles.committingDb.status=committing image/video database"}) - static class PrePopulateDataSourceFiles extends BulkTransferTask { - - /** - * @param dataSourceObjId The object ID of the DataSource that is being - * pre-populated into the DrawableDB. - * @param controller The controller for this task. - */ - PrePopulateDataSourceFiles(long dataSourceObjId, ImageGalleryController controller) { - super(dataSourceObjId, controller); - } - - @Override - protected void cleanup(boolean success) { - } - - @Override - void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDBTransaction) { - taskDB.insertBasicFileData(DrawableFile.create(f, false, false), tr, caseDBTransaction); - } - - @Override - @NbBundle.Messages({"PrePopulateDataSourceFiles.prepopulatingDb.status=prepopulating image/video database",}) - ProgressHandle getInitialProgressHandle() { - return ProgressHandle.createHandle(Bundle.PrePopulateDataSourceFiles_prepopulatingDb_status(), this); - } - } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 228b3765e5..88a7d38d70 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -43,11 +43,14 @@ import org.sleuthkit.autopsy.ingest.IngestManager.IngestJobEvent; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.DATA_ADDED; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.FILE_DONE; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisStartedEvent; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -281,8 +284,8 @@ public class ImageGalleryModule { //For a data source added on the local node, prepopulate all file data to drawable database if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { Content newDataSource = (Content) evt.getNewValue(); - if (con.isListeningEnabled()) { - con.queueDBTask(new ImageGalleryController.PrePopulateDataSourceFiles(newDataSource.getId(), controller)); + if (con.isListeningEnabled()) { + controller.getDatabase().insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.DEFAULT); } } break; @@ -326,38 +329,66 @@ public class ImageGalleryModule { @Override public void propertyChange(PropertyChangeEvent evt) { IngestJobEvent eventType = IngestJobEvent.valueOf(evt.getPropertyName()); - if (eventType != IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED - || ((AutopsyEvent) evt).getSourceType() != AutopsyEvent.SourceType.REMOTE) { - return; - } - // A remote node added a new data source and just finished ingest on it. - //drawable db is stale, and if ImageGallery is open, ask user what to do - + try { - ImageGalleryController con = getController(); - con.setStale(true); - if (con.isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { - SwingUtilities.invokeLater(() -> { - int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); - - switch (showAnswer) { - case JOptionPane.YES_OPTION: - con.rebuildDB(); - break; - case JOptionPane.NO_OPTION: - case JOptionPane.CANCEL_OPTION: - default: - break; //do nothing + ImageGalleryController controller = getController(); + + if (eventType == IngestJobEvent.DATA_SOURCE_ANALYSIS_STARTED) { + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + if (controller.isListeningEnabled()) { + DataSourceAnalysisStartedEvent dataSourceAnalysisStartedEvent = (DataSourceAnalysisStartedEvent) evt; + Content dataSource = dataSourceAnalysisStartedEvent.getDataSource(); + + controller.getDatabase().insertOrUpdateDataSource(dataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS); } - }); + } + } else if (eventType == IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED) { + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + if (controller.isListeningEnabled()) { + DataSourceAnalysisCompletedEvent dataSourceAnalysisCompletedEvent = (DataSourceAnalysisCompletedEvent) evt; + Content dataSource = dataSourceAnalysisCompletedEvent.getDataSource(); + + DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = + controller.hasFilesWithNoMimetype(dataSource) ? + DrawableDB.DrawableDbBuildStatusEnum.DEFAULT : + DrawableDB.DrawableDbBuildStatusEnum.COMPLETE; + + controller.getDatabase().insertOrUpdateDataSource(dataSource.getId(), datasourceDrawableDBStatus); + } + return; + } + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) { + // A remote node added a new data source and just finished ingest on it. + //drawable db is stale, and if ImageGallery is open, ask user what to do + controller.setStale(true); + if (controller.isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { + SwingUtilities.invokeLater(() -> { + int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + + switch (showAnswer) { + case JOptionPane.YES_OPTION: + controller.rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + default: + break; //do nothing + } + }); + } + } } - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS + } + catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error getting ImageGalleryController.", ex); //NON-NLS + logger.log(Level.SEVERE, "Error getting ImageGalleryController.", ex); //NON-NLS } } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index e51f27cdef..e9753daf5a 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -18,13 +18,10 @@ */ package org.sleuthkit.autopsy.imagegallery; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.function.Consumer; import java.util.logging.Level; import java.util.stream.Collectors; import javafx.application.Platform; @@ -55,10 +52,8 @@ import javafx.stage.Modality; import javax.swing.SwingUtilities; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.apache.commons.lang3.ObjectUtils.notEqual; -import org.apache.commons.lang3.StringUtils; import org.openide.explorer.ExplorerManager; import org.openide.explorer.ExplorerUtils; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; @@ -70,6 +65,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.gui.DataSourceCell; import org.sleuthkit.autopsy.imagegallery.gui.GuiUtils; @@ -232,8 +228,8 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl @SuppressWarnings(value = "unchecked") ComboBox> comboBox = (ComboBox>) datasourceDialog.getDialogPane().lookup(".combo-box"); //set custom cell renderer - comboBox.setCellFactory((ListView> param) -> new DataSourceCell(dataSourcesTooManyFiles)); - comboBox.setButtonCell(new DataSourceCell(dataSourcesTooManyFiles)); + comboBox.setCellFactory((ListView> param) -> new DataSourceCell(dataSourcesTooManyFiles, controller.getAllDataSourcesDrawableDBStatus())); + comboBox.setButtonCell(new DataSourceCell(dataSourcesTooManyFiles, controller.getAllDataSourcesDrawableDBStatus())); DataSource dataSource = datasourceDialog.showAndWait().orElse(Optional.empty()).orElse(null); try { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 81431b771e..230dd12867 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -19,11 +19,10 @@ package org.sleuthkit.autopsy.imagegallery.actions; import com.google.common.util.concurrent.ListenableFuture; -import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import java.awt.Component; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import static java.util.concurrent.Executors.newSingleThreadExecutor; +import java.util.Map; import java.util.logging.Level; import javafx.application.Platform; import javafx.scene.control.Alert; @@ -32,7 +31,6 @@ import javafx.scene.control.CheckBox; import javafx.scene.control.Label; import javafx.scene.layout.VBox; import javafx.stage.Modality; -import javax.annotation.concurrent.ThreadSafe; import javax.swing.ImageIcon; import javax.swing.JButton; import javax.swing.JMenuItem; @@ -50,11 +48,12 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.Installer; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.ImageGalleryController; import org.sleuthkit.autopsy.imagegallery.ImageGalleryModule; import org.sleuthkit.autopsy.imagegallery.ImageGalleryPreferences; import org.sleuthkit.autopsy.imagegallery.ImageGalleryTopComponent; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum; import org.sleuthkit.autopsy.imagegallery.gui.GuiUtils; import org.sleuthkit.autopsy.imagegallery.utils.TaskUtils; import static org.sleuthkit.autopsy.imagegallery.utils.TaskUtils.addFXCallback; @@ -70,6 +69,7 @@ import org.sleuthkit.datamodel.TskCoreException; "OpenAction.stale.confDlg.msg=The image / video database may be out of date. " + "Do you want to update and listen for further ingest results?\n" + "Choosing 'yes' will update the database and enable listening to future ingests.", + "OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet. Please run FileType and EXIF ingest modules.", "OpenAction.stale.confDlg.title=Image Gallery"}) public final class OpenAction extends CallableSystemAction { @@ -184,17 +184,43 @@ public final class OpenAction extends CallableSystemAction { } private void checkDBStale(ImageGalleryController controller) { - //check if db is stale on throw away bg thread and then react back on jfx thread. - ListenableFuture staleFuture = TaskUtils.getExecutorForClass(OpenAction.class) - .submit(controller::isDataSourcesTableStale); - addFXCallback(staleFuture, - dbIsStale -> { + + ListenableFuture> dataSourceStatusMapFuture = TaskUtils.getExecutorForClass(OpenAction.class) + .submit(controller::getAllDataSourcesDrawableDBStatus); + + addFXCallback(dataSourceStatusMapFuture, + dataSourceStatusMap -> { + + boolean dbIsStale = false; + for (Map.Entry entry : dataSourceStatusMap.entrySet()) { + DrawableDbBuildStatusEnum status = entry.getValue(); + if (DrawableDbBuildStatusEnum.COMPLETE != status) { + dbIsStale = true; + } + } + //back on fx thread. if (false == dbIsStale) { //drawable db is not stale, just open it openTopComponent(); } else { - //drawable db is stale, ask what to do + + // If there is only one datasource and it's in DEFAULT State - + // ingest modules need to be run on the data source + if (dataSourceStatusMap.size()== 1) { + Map.Entry entry = dataSourceStatusMap.entrySet().iterator().next(); + if (entry.getValue() == DrawableDbBuildStatusEnum.DEFAULT ) { + Alert alert = new Alert(Alert.AlertType.WARNING, Bundle.OpenAction_notAnalyzedDlg_msg(), ButtonType.OK); + alert.setTitle(Bundle.OpenAction_stale_confDlg_title()); + alert.initModality(Modality.APPLICATION_MODAL); + + alert.showAndWait(); + return; + } + } + + //drawable db is stale, + //ask what to do Alert alert = new Alert(Alert.AlertType.WARNING, Bundle.OpenAction_stale_confDlg_msg(), ButtonType.YES, ButtonType.NO, ButtonType.CANCEL); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 8c27f6cce5..dee725db34 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -174,11 +174,14 @@ public final class DrawableDB { /** * Enum to track Image gallery db rebuild status for a data source + * + * DO NOT add in the middle. */ public enum DrawableDbBuildStatusEnum { - UNKNOWN, /// no known status - IN_PROGRESS, /// drawable db rebuild has been started for the data source - COMPLETE; /// drawable db rebuild is complete for the data source + UNKNOWN, /// drawable db does not have an entry for this data source + IN_PROGRESS, /// drawable db rebuild has been started for the data source + COMPLETE, /// drawable db rebuild is complete for the data source + DEFAULT; /// drawable db needs to be updated for this data source } //////////////general database logic , mostly borrowed from sleuthkitcase diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java index 08543a42d2..76190ad1ef 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java @@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.imagegallery.gui; import java.util.Map; import java.util.Optional; import javafx.scene.control.ListCell; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum; import org.sleuthkit.datamodel.DataSource; /** @@ -29,9 +31,12 @@ import org.sleuthkit.datamodel.DataSource; public class DataSourceCell extends ListCell> { private final Map dataSourcesTooManyFiles; + private final Map dataSourcesDrawableDBStatus; - public DataSourceCell(Map dataSourcesViewable) { + public DataSourceCell(Map dataSourcesViewable, Map dataSourcesDrawableDBStatus) { this.dataSourcesTooManyFiles = dataSourcesViewable; + this.dataSourcesDrawableDBStatus = dataSourcesDrawableDBStatus; + } @Override @@ -43,14 +48,28 @@ public class DataSourceCell extends ListCell> { DataSource dataSource = item.orElse(null); String text = (dataSource == null) ? "All" : dataSource.getName() + " (Id: " + dataSource.getId() + ")"; Boolean tooManyFilesInDataSource = dataSourcesTooManyFiles.getOrDefault(dataSource, false); + + DrawableDbBuildStatusEnum dataSourceDBStatus = (dataSource != null) ? + dataSourcesDrawableDBStatus.get(dataSource.getId()) : DrawableDbBuildStatusEnum.UNKNOWN; + + Boolean dataSourceNotAnalyzed = (dataSourceDBStatus == DrawableDbBuildStatusEnum.DEFAULT); if (tooManyFilesInDataSource) { text += " - Too many files"; + } + if (dataSourceNotAnalyzed) { + text += " - Not Analyzed"; + } + + // check if item should be disabled + if (tooManyFilesInDataSource || dataSourceNotAnalyzed) { + setDisable(true); setStyle("-fx-opacity : .5"); - } else { + } + else { setGraphic(null); setStyle("-fx-opacity : 1"); } - setDisable(tooManyFilesInDataSource); + setText(text); } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java index bc0f5e5d84..4bcecbbecb 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java @@ -240,8 +240,8 @@ public class Toolbar extends ToolBar { } private void initDataSourceComboBox() { - dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable)); - dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable)); + dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus())); + dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus())); dataSourceComboBox.setConverter(new StringConverter>() { @Override public String toString(Optional object) { From 503e1e56ef760157fc3ce67743c9f3436d0ecb87 Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 13 Nov 2018 18:40:40 -0500 Subject: [PATCH 13/14] Manually applied changes to resolve merge conflict - Push isImageGalleryOpen() check on EDT. --- .../imagegallery/ImageGalleryModule.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 8386999c69..2dd4ba2362 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -364,21 +364,23 @@ public class ImageGalleryModule { // A remote node added a new data source and just finished ingest on it. //drawable db is stale, and if ImageGallery is open, ask user what to do controller.setStale(true); - if (controller.isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { + if (controller.isListeningEnabled()) { SwingUtilities.invokeLater(() -> { - int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + if (ImageGalleryTopComponent.isImageGalleryOpen()) { + int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); - switch (showAnswer) { - case JOptionPane.YES_OPTION: - controller.rebuildDB(); - break; - case JOptionPane.NO_OPTION: - case JOptionPane.CANCEL_OPTION: - default: - break; //do nothing + switch (showAnswer) { + case JOptionPane.YES_OPTION: + controller.rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + default: + break; //do nothing + } } }); } From 8d2326b475b00d9c618b4b43e42915eb9ad7798e Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 14 Nov 2018 10:21:12 -0500 Subject: [PATCH 14/14] Address review comments and Codacy comments in the previous commit. --- .../imagegallery/ImageGalleryController.java | 3 +- .../imagegallery/ImageGalleryModule.java | 1 - .../ImageGalleryTopComponent.java | 1 - .../imagegallery/actions/OpenAction.java | 3 +- .../imagegallery/datamodel/DrawableDB.java | 8 +- .../imagegallery/gui/DataSourceCell.java | 15 +- release/update_autopsy_version.pl | 268 ++++++++++++++++++ release/update_sleuthkit_version.pl | 199 +++++++++++++ 8 files changed, 488 insertions(+), 10 deletions(-) create mode 100755 release/update_autopsy_version.pl create mode 100755 release/update_sleuthkit_version.pl diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index bc78be5fd7..6881bb2e88 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -450,7 +450,8 @@ public final class ImageGalleryController { public boolean hasFilesWithNoMimetype(Content datasource) throws TskCoreException { // There are some special files/attributes in the root folder, like $BadClus:$Bad and $Security:$SDS - // that do not go through any ingest modules, and hence do not have any assigned mimetype + // The IngestTasksScheduler does not push them down to the ingest modules, + // and hence they do not have any assigned mimetype String whereClause = "data_source_obj_id = " + datasource.getId() + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + " AND ( mime_type IS NULL )" diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 2dd4ba2362..530e3eb7d6 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -50,7 +50,6 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index e9753daf5a..927dc6b70e 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -65,7 +65,6 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute; -import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.gui.DataSourceCell; import org.sleuthkit.autopsy.imagegallery.gui.GuiUtils; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 230dd12867..5a921e6130 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -69,7 +69,8 @@ import org.sleuthkit.datamodel.TskCoreException; "OpenAction.stale.confDlg.msg=The image / video database may be out of date. " + "Do you want to update and listen for further ingest results?\n" + "Choosing 'yes' will update the database and enable listening to future ingests.", - "OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet. Please run FileType and EXIF ingest modules.", + "OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet.\n" + + "Please run FileType and EXIF ingest modules.", "OpenAction.stale.confDlg.title=Image Gallery"}) public final class OpenAction extends CallableSystemAction { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index dee725db34..13eea6d654 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -178,10 +178,10 @@ public final class DrawableDB { * DO NOT add in the middle. */ public enum DrawableDbBuildStatusEnum { - UNKNOWN, /// drawable db does not have an entry for this data source - IN_PROGRESS, /// drawable db rebuild has been started for the data source - COMPLETE, /// drawable db rebuild is complete for the data source - DEFAULT; /// drawable db needs to be updated for this data source + UNKNOWN, /// no known status + IN_PROGRESS, /// ingest or db rebuild is in progress + COMPLETE, /// All files in the data source have had file type detected + DEFAULT; /// Not all files in the data source have had file type detected } //////////////general database logic , mostly borrowed from sleuthkitcase diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java index 76190ad1ef..7626c4fd77 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java @@ -33,8 +33,19 @@ public class DataSourceCell extends ListCell> { private final Map dataSourcesTooManyFiles; private final Map dataSourcesDrawableDBStatus; - public DataSourceCell(Map dataSourcesViewable, Map dataSourcesDrawableDBStatus) { - this.dataSourcesTooManyFiles = dataSourcesViewable; + /** + * + * @param dataSourcesTooManyFiles: a map of too many files indicator for + * each data source. + * Data sources with too many files may substantially slow down + * the system and hence are disabled for selection. + * @param dataSourcesDrawableDBStatus a map of drawable DB status for + * each data sources. + * Data sources in DEFAULT state are not fully analyzed yet and are + * disabled for selection. + */ + public DataSourceCell(Map dataSourcesTooManyFiles, Map dataSourcesDrawableDBStatus) { + this.dataSourcesTooManyFiles = dataSourcesTooManyFiles; this.dataSourcesDrawableDBStatus = dataSourcesDrawableDBStatus; } diff --git a/release/update_autopsy_version.pl b/release/update_autopsy_version.pl new file mode 100755 index 0000000000..0a115c8daf --- /dev/null +++ b/release/update_autopsy_version.pl @@ -0,0 +1,268 @@ +#!/usr/bin/perl + +# Updates various Autopsy version numbers + +use strict; +use File::Copy; + +# global variables +my $VER; + + +my $TESTING = 0; +print "TESTING MODE (no commits)\n" if ($TESTING); + + + +sub main { + + # Get the Autopsy version argument + if (scalar (@ARGV) != 1) { + print stderr "Missing release version argument (i.e. 4.9.0)\n"; + exit; + } + + $VER = $ARGV[0]; + die "Invalid version number: $VER (1.2.3 or 1.2.3b1 expected)" unless ($VER =~ /^\d+\.\d+\.\d+(b\d+)?$/); + + + my $AUT_RELNAME = "autopsy-${VER}"; + # Verify the tag doesn't already exist + exec_pipe(*OUT, "git tag | grep \"${AUT_RELNAME}\$\""); + my $foo = read_pipe_line(*OUT); + if ($foo ne "") { + print "Tag ${AUT_RELNAME} already exists\n"; + print "Remove with 'git tag -d ${AUT_RELNAME}'\n"; + die "stopping"; + } + close(OUT); + + # Assume we running out of 'release' folder + chdir ".." or die "Error changing directories to root"; + + + # verify_precheckin(); + + + # Update the version info in that tag + update_project_properties(); + update_doxygen_dev(); + update_doxygen_user(); + + print "Files updated. You need to commit and push them\n"; +} + + + + + +###################################################### +# Utility functions + + +# Function to execute a command and send output to pipe +# returns handle +# exec_pipe(HANDLE, CMD); +sub exec_pipe { + my $handle = shift(@_); + my $cmd = shift(@_); + + die "Can't open pipe for exec_pipe" + unless defined(my $pid = open($handle, '-|')); + + if ($pid) { + return $handle; + } + else { + $| = 1; + exec("$cmd") or die "Can't exec program: $!"; + } +} + +# Read a line of text from an open exec_pipe handle +sub read_pipe_line { + my $handle = shift(@_); + my $out; + + for (my $i = 0; $i < 100; $i++) { + $out = <$handle>; + return $out if (defined $out); + } + return $out; +} + + +# Prompt user for argument and return response +sub prompt_user { + my $q = shift(@_); + print "$q: "; + $| = 1; + $_ = ; + chomp; + return $_; +} + + + +############################################# +# File update methods + + + +# Verify that all files in the current source directory +# are checked in. dies if any are modified. +sub verify_precheckin { + + #system ("git pull"); + + print "Verifying everything is checked in\n"; + exec_pipe(*OUT, "git status -s | grep \"^ M\""); + + my $foo = read_pipe_line(*OUT); + if ($foo ne "") { + print "Files not checked in\n"; + while ($foo ne "") { + print "$foo"; + $foo = read_pipe_line(*OUT); + } + die "stopping" unless ($TESTING); + } + close(OUT); + + print "Verifying everything is pushed\n"; + exec_pipe(*OUT, "git status -sb | grep \"^##\" | grep \"ahead \""); + my $foo = read_pipe_line(*OUT); + if ($foo ne "") { + print "$foo"; + print "Files not pushed to remote\n"; + die "stopping" unless ($TESTING); + } + close(OUT); +} + + + +# update the version in nbproject/project.properties +sub update_project_properties { + + my $orig = "project.properties"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + chdir "nbproject" or die "cannot change into nbproject directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^app\.version=/) { + print CONF_OUT "app.version=$VER\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 1) { + die "$found (instead of 1) occurrences of app.version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir ".." or die "Error changing directories back to root"; +} + + + +# update the dev docs +sub update_doxygen_dev { + + my $orig = "Doxyfile"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig} (Dev)\n"; + + chdir "docs/doxygen" or die "cannot change into docs/doxygen directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^PROJECT_NUMBER/) { + print CONF_OUT "PROJECT_NUMBER = ${VER}\n"; + $found++; + } + elsif (/^HTML_OUTPUT/) { + print CONF_OUT "HTML_OUTPUT = api-docs/${VER}/\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 2) { + die "$found (instead of 2) occurrences of version found in (DEV) ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + + +# update the user docs +sub update_doxygen_user { + + my $orig = "Doxyfile"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig} (User)\n"; + + chdir "docs/doxygen-user" or die "cannot change into docs/doxygen-user directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^PROJECT_NUMBER/) { + print CONF_OUT "PROJECT_NUMBER = ${VER}\n"; + $found++; + } + elsif (/^HTML_OUTPUT/) { + print CONF_OUT "HTML_OUTPUT = ${VER}\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 2) { + die "$found (instead of 2) occurrences of version found in (USER) ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + + +main(); \ No newline at end of file diff --git a/release/update_sleuthkit_version.pl b/release/update_sleuthkit_version.pl new file mode 100755 index 0000000000..e630e4890b --- /dev/null +++ b/release/update_sleuthkit_version.pl @@ -0,0 +1,199 @@ +#!/usr/bin/perl + +# Updates various TSK version numbers +# use this when the version of TSK that Autopsy depends on changes + +use strict; +use File::Copy; + +# global variables +my $VER; + +my $TESTING = 0; +print "TESTING MODE (no commits)\n" if ($TESTING); + + +sub main { + + # Get the TSK version argument + if (scalar (@ARGV) != 1) { + print stderr "Missing release version argument (i.e. 4.9.0)\n"; + exit; + } + + $VER = $ARGV[0]; + die "Invalid version number: $VER (1.2.3 or 1.2.3b1 expected)" unless ($VER =~ /^\d+\.\d+\.\d+(b\d+)?$/); + + # Assume we running out of 'release' folder + chdir ".." or die "Error changing directories to root"; + + # Update the version info in that tag + update_tsk_version(); + update_core_project_properties(); + update_core_project_xml(); + + print "Files updated. You need to commit and push them\n"; +} + + + + + +###################################################### +# Utility functions + + +# Function to execute a command and send output to pipe +# returns handle +# exec_pipe(HANDLE, CMD); +sub exec_pipe { + my $handle = shift(@_); + my $cmd = shift(@_); + + die "Can't open pipe for exec_pipe" + unless defined(my $pid = open($handle, '-|')); + + if ($pid) { + return $handle; + } + else { + $| = 1; + exec("$cmd") or die "Can't exec program: $!"; + } +} + +# Read a line of text from an open exec_pipe handle +sub read_pipe_line { + my $handle = shift(@_); + my $out; + + for (my $i = 0; $i < 100; $i++) { + $out = <$handle>; + return $out if (defined $out); + } + return $out; +} + + + +############################################# +# File update methods + + + +# update the tskversion.xml +sub update_tsk_version { + + my $orig = "TSKVersion.xml"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/name="TSK_VERSION" value=/) { + print CONF_OUT " \n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 1) { + die "$found (instead of 1) occurrences of app.version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + +} + + + +sub update_core_project_properties { + + my $orig = "project.properties"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + chdir "Core/nbproject" or die "cannot change into Core/nbproject directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^file\.reference\.sleuthkit\-postgresql-/) { + print CONF_OUT "file.reference.sleuthkit-postgresql-${VER}.jar=release/modules/ext/sleuthkit-postgresql-${VER}.jar\n"; + $found++; + } + + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 1) { + die "$found (instead of 1) occurrences of version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + +sub update_core_project_xml { + + my $orig = "project.xml"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + chdir "Core/nbproject" or die "cannot change into Core/nbproject directory"; + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/ext\/sleuthkit-postgresql/) { + print CONF_OUT " ext/sleuthkit-postgresql-${VER}.jar\n"; + $found++; + } + elsif (/release\/modules\/ext\/sleuthkit-postgresql/) { + print CONF_OUT " release/modules/ext/sleuthkit-postgresql-${VER}.jar\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 2) { + die "$found (instead of 2) occurrences of version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + + + + +main(); \ No newline at end of file