diff --git a/.gitattributes b/.gitattributes index 6cd046f17b..cd5271c982 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,7 +1,7 @@ *.java text diff=java *.txt text -*.sh text +*.sh eol=lf *.mf text *.xml text *.form text diff --git a/.travis.yml b/.travis.yml index 7554bbc6a4..56c8b7bbbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,10 +64,3 @@ script: - cd $TRAVIS_BUILD_DIR/ - ant build - echo -en 'travis_fold:end:script.build\\r' - - echo "Testing Autopsy..." && echo -en 'travis_fold:start:script.tests\\r' - - cd Core/ - - ant -q getTestDataFiles - - echo "Free Space:" - - echo `df -h .` - - xvfb-run ant -q test - - echo -en 'travis_fold:end:script.tests\\r' diff --git a/Core/ivy.xml b/Core/ivy.xml index 54ed532feb..9ba4a3c371 100644 --- a/Core/ivy.xml +++ b/Core/ivy.xml @@ -43,6 +43,8 @@ + + diff --git a/Core/manifest.mf b/Core/manifest.mf index 53b737fbd1..dbfb89dc59 100644 --- a/Core/manifest.mf +++ b/Core/manifest.mf @@ -2,7 +2,7 @@ Manifest-Version: 1.0 OpenIDE-Module: org.sleuthkit.autopsy.core/10 OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/core/Bundle.properties OpenIDE-Module-Layer: org/sleuthkit/autopsy/core/layer.xml -OpenIDE-Module-Implementation-Version: 28 +OpenIDE-Module-Implementation-Version: 29 OpenIDE-Module-Requires: org.openide.windows.WindowManager AutoUpdate-Show-In-Client: true AutoUpdate-Essential-Module: true diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index d9f4d9824a..2a952926ba 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -3,6 +3,7 @@ file.reference.apache-mime4j-core-0.8.2.jar=release\\modules\\ext\\apache-mime4j file.reference.apache-mime4j-dom-0.8.2.jar=release\\modules\\ext\\apache-mime4j-dom-0.8.2.jar file.reference.asm-7.0.jar=release\\modules\\ext\\asm-7.0.jar file.reference.bcmail-jdk15on-1.60.jar=release\\modules\\ext\\bcmail-jdk15on-1.60.jar +file.reference.bcpkix-jdk15on-1.60.jar=release\\modules\\ext\\bcpkix-jdk15on-1.60.jar file.reference.bcprov-jdk15on-1.60.jar=release\\modules\\ext\\bcprov-jdk15on-1.60.jar file.reference.boilerpipe-1.1.0.jar=release\\modules\\ext\\boilerpipe-1.1.0.jar file.reference.c3p0-0.9.5.jar=release/modules/ext/c3p0-0.9.5.jar @@ -16,7 +17,6 @@ file.reference.commons-io-2.6.jar=release\\modules\\ext\\commons-io-2.6.jar file.reference.commons-lang3-3.8.1.jar=release\\modules\\ext\\commons-lang3-3.8.1.jar file.reference.commons-pool2-2.4.2.jar=release/modules/ext/commons-pool2-2.4.2.jar file.reference.cxf-rt-rs-client-3.3.0.jar=release\\modules\\ext\\cxf-rt-rs-client-3.3.0.jar -file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar file.reference.dec-0.1.2.jar=release\\modules\\ext\\dec-0.1.2.jar file.reference.fontbox-2.0.13.jar=release\\modules\\ext\\fontbox-2.0.13.jar file.reference.geoapi-3.0.1.jar=release\\modules\\ext\\geoapi-3.0.1.jar @@ -55,7 +55,7 @@ file.reference.mchange-commons-java-0.2.9.jar=release/modules/ext/mchange-common file.reference.metadata-extractor-2.11.0.jar=release\\modules\\ext\\metadata-extractor-2.11.0.jar file.reference.netcdf4-4.5.5.jar=release\\modules\\ext\\netcdf4-4.5.5.jar file.reference.openjson-1.0.10.jar=release\\modules\\ext\\openjson-1.0.10.jar -file.reference.opennlp-tools-1.9.0.jar=release\\modules\\ext\\opennlp-tools-1.9.0.jar +file.reference.opennlp-tools-1.9.1.jar=release\\modules\\ext\\opennlp-tools-1.9.1.jar file.reference.parso-2.0.10.jar=release\\modules\\ext\\parso-2.0.10.jar file.reference.pdfbox-2.0.13.jar=release\\modules\\ext\\pdfbox-2.0.13.jar file.reference.pdfbox-tools-2.0.13.jar=release\\modules\\ext\\pdfbox-tools-2.0.13.jar @@ -75,7 +75,7 @@ file.reference.javax.ws.rs-api-2.0.1.jar=release/modules/ext/javax.ws.rs-api-2.0 file.reference.cxf-core-3.0.16.jar=release/modules/ext/cxf-core-3.0.16.jar file.reference.cxf-rt-frontend-jaxrs-3.0.16.jar=release/modules/ext/cxf-rt-frontend-jaxrs-3.0.16.jar file.reference.cxf-rt-transports-http-3.0.16.jar=release/modules/ext/cxf-rt-transports-http-3.0.16.jar -file.reference.sleuthkit-postgresql-4.6.7.jar=release/modules/ext/sleuthkit-postgresql-4.6.7.jar +file.reference.sleuthkit-postgresql-4.7.0.jar=release/modules/ext/sleuthkit-postgresql-4.7.0.jar file.reference.curator-client-2.8.0.jar=release/modules/ext/curator-client-2.8.0.jar file.reference.curator-framework-2.8.0.jar=release/modules/ext/curator-framework-2.8.0.jar file.reference.curator-recipes-2.8.0.jar=release/modules/ext/curator-recipes-2.8.0.jar @@ -86,6 +86,7 @@ file.reference.uimafit-core-2.4.0.jar=release\\modules\\ext\\uimafit-core-2.4.0. file.reference.uimaj-core-3.0.1.jar=release\\modules\\ext\\uimaj-core-3.0.1.jar file.reference.vorbis-java-core-0.8.jar=release\\modules\\ext\\vorbis-java-core-0.8.jar file.reference.vorbis-java-tika-0.8.jar=release\\modules\\ext\\vorbis-java-tika-0.8.jar +file.reference.webp-imageio-sejda-0.1.0.jar=release/modules/ext/webp-imageio-sejda-0.1.0.jar file.reference.xmlbeans-3.0.2.jar=release\\modules\\ext\\xmlbeans-3.0.2.jar file.reference.xmpcore-5.1.3.jar=release/modules/ext/xmpcore-5.1.3.jar file.reference.xz-1.8.jar=release\\modules\\ext\\xz-1.8.jar @@ -122,5 +123,5 @@ nbm.homepage=http://www.sleuthkit.org/ nbm.module.author=Brian Carrier nbm.needs.restart=true source.reference.curator-recipes-2.8.0.jar=release/modules/ext/curator-recipes-2.8.0-sources.jar -spec.version.base=10.16 +spec.version.base=10.17 diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 967399cc20..c33dabfbc7 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -251,7 +251,7 @@ 3 - 1.2 + 1.3 @@ -350,49 +350,29 @@ release\modules\ext\commons-lang3-3.8.1.jar - ext/jempbox-1.8.16.jar - release\modules\ext\jempbox-1.8.16.jar - - - ext/jackcess-2.2.0.jar - release\modules\ext\jackcess-2.2.0.jar - - - ext/jericho-html-3.3.jar - release/modules/ext/jericho-html-3.3.jar + ext/gax-grpc-1.44.0.jar + release/modules/ext/gax-grpc-1.44.0.jar ext/cdm-4.5.5.jar release\modules\ext\cdm-4.5.5.jar - - ext/httpservices-4.5.5.jar - release\modules\ext\httpservices-4.5.5.jar - - - ext/xz-1.8.jar - release\modules\ext\xz-1.8.jar - - - ext/commons-validator-1.6.jar - release/modules/ext/commons-validator-1.6.jar - ext/sis-utility-0.8.jar release\modules\ext\sis-utility-0.8.jar - ext/jna-5.1.0.jar - release\modules\ext\jna-5.1.0.jar + ext/opencensus-api-0.19.2.jar + release/modules/ext/opencensus-api-0.19.2.jar + + + ext/gax-httpjson-0.61.0.jar + release/modules/ext/gax-httpjson-0.61.0.jar ext/boilerpipe-1.1.0.jar release\modules\ext\boilerpipe-1.1.0.jar - - ext/jbig2-imageio-3.0.2.jar - release\modules\ext\jbig2-imageio-3.0.2.jar - ext/jsoup-1.11.3.jar release\modules\ext\jsoup-1.11.3.jar @@ -401,22 +381,10 @@ ext/sevenzipjbinding.jar release/modules/ext/sevenzipjbinding.jar - - ext/apache-mime4j-dom-0.8.2.jar - release\modules\ext\apache-mime4j-dom-0.8.2.jar - ext/mchange-commons-java-0.2.9.jar release/modules/ext/mchange-commons-java-0.2.9.jar - - ext/pdfbox-2.0.13.jar - release\modules\ext\pdfbox-2.0.13.jar - - - ext/xmlbeans-3.0.2.jar - release\modules\ext\xmlbeans-3.0.2.jar - ext/jackson-databind-2.9.7.jar release\modules\ext\jackson-databind-2.9.7.jar @@ -425,41 +393,33 @@ ext/jai-imageio-core-1.4.0.jar release\modules\ext\jai-imageio-core-1.4.0.jar + + ext/api-common-1.7.0.jar + release/modules/ext/api-common-1.7.0.jar + ext/jcl-over-slf4j-1.7.25.jar release\modules\ext\jcl-over-slf4j-1.7.25.jar - ext/curator-recipes-2.8.0.jar - release/modules/ext/curator-recipes-2.8.0.jar + ext/okhttp-2.7.5.jar + release/modules/ext/okhttp-2.7.5.jar ext/tika-core-1.20.jar release\modules\ext\tika-core-1.20.jar - - ext/tagsoup-1.2.1.jar - release\modules\ext\tagsoup-1.2.1.jar - ext/StixLib.jar release/modules/ext/StixLib.jar - - ext/jackson-core-2.9.7.jar - release\modules\ext\jackson-core-2.9.7.jar - - - ext/sis-metadata-0.8.jar - release\modules\ext\sis-metadata-0.8.jar - ext/bcprov-jdk15on-1.60.jar release\modules\ext\bcprov-jdk15on-1.60.jar - ext/parso-2.0.10.jar - release\modules\ext\parso-2.0.10.jar + ext/google-auth-library-credentials-0.15.0.jar + release/modules/ext/google-auth-library-credentials-0.15.0.jar ext/json-simple-1.1.1.jar @@ -473,18 +433,10 @@ ext/commons-codec-1.11.jar release\modules\ext\commons-codec-1.11.jar - - ext/apache-mime4j-core-0.8.2.jar - release\modules\ext\apache-mime4j-core-0.8.2.jar - ext/jmatio-1.5.jar release\modules\ext\jmatio-1.5.jar - - ext/sleuthkit-postgresql-4.6.7.jar - release/modules/ext/sleuthkit-postgresql-4.6.7.jar - ext/tika-parsers-1.20.jar release\modules\ext\tika-parsers-1.20.jar @@ -497,18 +449,10 @@ ext/commons-pool2-2.4.2.jar release/modules/ext/commons-pool2-2.4.2.jar - - ext/commons-io-2.6.jar - release\modules\ext\commons-io-2.6.jar - ext/jdom-2.0.5-contrib.jar release/modules/ext/jdom-2.0.5-contrib.jar - - ext/SparseBitSet-1.1.jar - release/modules/ext/SparseBitSet-1.1.jar - ext/openjson-1.0.10.jar release\modules\ext\openjson-1.0.10.jar @@ -517,38 +461,18 @@ ext/isoparser-1.1.22.jar release\modules\ext\isoparser-1.1.22.jar - - ext/c3p0-0.9.5.jar - release/modules/ext/c3p0-0.9.5.jar - ext/xmpcore-5.1.3.jar release/modules/ext/xmpcore-5.1.3.jar - - ext/zookeeper-3.4.6.jar - release/modules/ext/zookeeper-3.4.6.jar - ext/javax.activation-1.2.0.jar release\modules\ext\javax.activation-1.2.0.jar - - ext/commons-csv-1.6.jar - release\modules\ext\commons-csv-1.6.jar - - - ext/jdom-2.0.5.jar - release/modules/ext/jdom-2.0.5.jar - ext/rome-1.12.0.jar release\modules\ext\rome-1.12.0.jar - - ext/jackson-annotations-2.9.7.jar - release\modules\ext\jackson-annotations-2.9.7.jar - ext/javax.annotation-api-1.3.2.jar release\modules\ext\javax.annotation-api-1.3.2.jar @@ -557,49 +481,25 @@ ext/vorbis-java-core-0.8.jar release\modules\ext\vorbis-java-core-0.8.jar - - ext/netcdf4-4.5.5.jar - release\modules\ext\netcdf4-4.5.5.jar - ext/java-libpst-0.8.1.jar release\modules\ext\java-libpst-0.8.1.jar - ext/opennlp-tools-1.9.0.jar - release\modules\ext\opennlp-tools-1.9.0.jar - - - ext/sis-netcdf-0.8.jar - release\modules\ext\sis-netcdf-0.8.jar + ext/okio-1.6.0.jar + release/modules/ext/okio-1.6.0.jar ext/curator-framework-2.8.0.jar release/modules/ext/curator-framework-2.8.0.jar - - ext/sentiment-analysis-parser-0.1.jar - release\modules\ext\sentiment-analysis-parser-0.1.jar - - - ext/commons-collections4-4.2.jar - release\modules\ext\commons-collections4-4.2.jar - ext/commons-dbcp2-2.1.1.jar release/modules/ext/commons-dbcp2-2.1.1.jar - ext/jgraphx-v3.8.0.jar - release/modules/ext/jgraphx-v3.8.0.jar - - - ext/juniversalchardet-1.0.3.jar - release\modules\ext\juniversalchardet-1.0.3.jar - - - ext/jython-standalone-2.7.0.jar - release/modules/ext/jython-standalone-2.7.0.jar + ext/google-http-client-appengine-1.29.0.jar + release/modules/ext/google-http-client-appengine-1.29.0.jar ext/uimafit-core-2.4.0.jar @@ -609,26 +509,30 @@ ext/jackcess-encrypt-2.1.4.jar release\modules\ext\jackcess-encrypt-2.1.4.jar - - ext/jhighlight-1.0.3.jar - release\modules\ext\jhighlight-1.0.3.jar - ext/junrar-2.0.0.jar release\modules\ext\junrar-2.0.0.jar - ext/jul-to-slf4j-1.7.25.jar - release\modules\ext\jul-to-slf4j-1.7.25.jar + ext/google-http-client-1.29.0.jar + release/modules/ext/google-http-client-1.29.0.jar - ext/postgresql-9.4.1211.jre7.jar - release/modules/ext/postgresql-9.4.1211.jre7.jar + ext/bcpkix-jdk15on-1.60.jar + release\modules\ext\bcpkix-jdk15on-1.60.jar + + + ext/opennlp-tools-1.9.1.jar + release\modules\ext\opennlp-tools-1.9.1.jar ext/slf4j-api-1.7.25.jar release\modules\ext\slf4j-api-1.7.25.jar + + ext/google-cloud-core-1.70.0.jar + release/modules/ext/google-cloud-core-1.70.0.jar + ext/geoapi-3.0.1.jar release\modules\ext\geoapi-3.0.1.jar @@ -641,18 +545,10 @@ ext/jdom2-2.0.6.jar release\modules\ext\jdom2-2.0.6.jar - - ext/httpclient-4.5.6.jar - release\modules\ext\httpclient-4.5.6.jar - ext/uimaj-core-3.0.1.jar release\modules\ext\uimaj-core-3.0.1.jar - - ext/curator-client-2.8.0.jar - release/modules/ext/curator-client-2.8.0.jar - ext/sqlite-jdbc-3.25.2.jar release/modules/ext/sqlite-jdbc-3.25.2.jar @@ -670,65 +566,133 @@ release\modules\ext\grib-4.5.5.jar - ext/fontbox-2.0.13.jar - release\modules\ext\fontbox-2.0.13.jar + ext/gax-1.44.0.jar + release/modules/ext/gax-1.44.0.jar - ext/activemq-all-5.11.1.jar - release/modules/ext/activemq-all-5.11.1.jar + ext/jempbox-1.8.16.jar + release\modules\ext\jempbox-1.8.16.jar - ext/dec-0.1.2.jar - release\modules\ext\dec-0.1.2.jar - - - ext/Rejistry-1.1-SNAPSHOT.jar - release/modules/ext/Rejistry-1.1-SNAPSHOT.jar - - - ext/dd-plist-1.20.jar - release/modules/ext/dd-plist-1.20.jar - - - ext/sevenzipjbinding-AllPlatforms.jar - release/modules/ext/sevenzipjbinding-AllPlatforms.jar - - - ext/bcmail-jdk15on-1.60.jar - release\modules\ext\bcmail-jdk15on-1.60.jar - - - ext/vorbis-java-tika-0.8.jar - release\modules\ext\vorbis-java-tika-0.8.jar + ext/jackcess-2.2.0.jar + release\modules\ext\jackcess-2.2.0.jar ext/grpc-context-1.19.0.jar release/modules/ext/grpc-context-1.19.0.jar - ext/gax-grpc-1.44.0.jar - release/modules/ext/gax-grpc-1.44.0.jar + ext/jericho-html-3.3.jar + release/modules/ext/jericho-html-3.3.jar - ext/opencensus-api-0.19.2.jar - release/modules/ext/opencensus-api-0.19.2.jar + ext/httpservices-4.5.5.jar + release\modules\ext\httpservices-4.5.5.jar - ext/gax-httpjson-0.61.0.jar - release/modules/ext/gax-httpjson-0.61.0.jar + ext/xz-1.8.jar + release\modules\ext\xz-1.8.jar - ext/api-common-1.7.0.jar - release/modules/ext/api-common-1.7.0.jar + ext/commons-validator-1.6.jar + release/modules/ext/commons-validator-1.6.jar - ext/google-auth-library-credentials-0.15.0.jar - release/modules/ext/google-auth-library-credentials-0.15.0.jar + ext/jna-5.1.0.jar + release\modules\ext\jna-5.1.0.jar + + + ext/jbig2-imageio-3.0.2.jar + release\modules\ext\jbig2-imageio-3.0.2.jar + + + ext/sleuthkit-postgresql-4.7.0.jar + release/modules/ext/sleuthkit-postgresql-4.7.0.jar + + + ext/apache-mime4j-dom-0.8.2.jar + release\modules\ext\apache-mime4j-dom-0.8.2.jar + + + ext/pdfbox-2.0.13.jar + release\modules\ext\pdfbox-2.0.13.jar + + + ext/xmlbeans-3.0.2.jar + release\modules\ext\xmlbeans-3.0.2.jar + + + ext/curator-recipes-2.8.0.jar + release/modules/ext/curator-recipes-2.8.0.jar + + + ext/tagsoup-1.2.1.jar + release\modules\ext\tagsoup-1.2.1.jar + + + ext/jackson-core-2.9.7.jar + release\modules\ext\jackson-core-2.9.7.jar + + + ext/sis-metadata-0.8.jar + release\modules\ext\sis-metadata-0.8.jar + + + ext/parso-2.0.10.jar + release\modules\ext\parso-2.0.10.jar + + + ext/apache-mime4j-core-0.8.2.jar + release\modules\ext\apache-mime4j-core-0.8.2.jar + + + ext/commons-io-2.6.jar + release\modules\ext\commons-io-2.6.jar + + + ext/SparseBitSet-1.1.jar + release/modules/ext/SparseBitSet-1.1.jar + + + ext/c3p0-0.9.5.jar + release/modules/ext/c3p0-0.9.5.jar + + + ext/zookeeper-3.4.6.jar + release/modules/ext/zookeeper-3.4.6.jar + + + ext/commons-csv-1.6.jar + release\modules\ext\commons-csv-1.6.jar + + + ext/jdom-2.0.5.jar + release/modules/ext/jdom-2.0.5.jar + + + ext/jackson-annotations-2.9.7.jar + release\modules\ext\jackson-annotations-2.9.7.jar ext/google-api-client-1.27.0.jar release/modules/ext/google-api-client-1.27.0.jar + + ext/netcdf4-4.5.5.jar + release\modules\ext\netcdf4-4.5.5.jar + + + ext/sis-netcdf-0.8.jar + release\modules\ext\sis-netcdf-0.8.jar + + + ext/sentiment-analysis-parser-0.1.jar + release\modules\ext\sentiment-analysis-parser-0.1.jar + + + ext/commons-collections4-4.2.jar + release\modules\ext\commons-collections4-4.2.jar + ext/opencensus-contrib-http-util-0.19.2.jar release/modules/ext/opencensus-contrib-http-util-0.19.2.jar @@ -738,21 +702,57 @@ release/modules/ext/google-auth-library-oauth2-http-0.15.0.jar - ext/google-http-client-appengine-1.29.0.jar - release/modules/ext/google-http-client-appengine-1.29.0.jar + ext/jgraphx-v3.8.0.jar + release/modules/ext/jgraphx-v3.8.0.jar - ext/google-http-client-1.29.0.jar - release/modules/ext/google-http-client-1.29.0.jar + ext/juniversalchardet-1.0.3.jar + release\modules\ext\juniversalchardet-1.0.3.jar - ext/google-cloud-core-1.70.0.jar - release/modules/ext/google-cloud-core-1.70.0.jar + ext/jython-standalone-2.7.0.jar + release/modules/ext/jython-standalone-2.7.0.jar + + + ext/jhighlight-1.0.3.jar + release\modules\ext\jhighlight-1.0.3.jar + + + ext/jul-to-slf4j-1.7.25.jar + release\modules\ext\jul-to-slf4j-1.7.25.jar + + + ext/postgresql-9.4.1211.jre7.jar + release/modules/ext/postgresql-9.4.1211.jre7.jar + + + ext/httpclient-4.5.6.jar + release\modules\ext\httpclient-4.5.6.jar + + + ext/curator-client-2.8.0.jar + release/modules/ext/curator-client-2.8.0.jar + + + ext/fontbox-2.0.13.jar + release\modules\ext\fontbox-2.0.13.jar + + + ext/activemq-all-5.11.1.jar + release/modules/ext/activemq-all-5.11.1.jar ext/google-cloud-core-http-1.70.0.jar release/modules/ext/google-cloud-core-http-1.70.0.jar + + ext/Rejistry-1.1-SNAPSHOT.jar + release/modules/ext/Rejistry-1.1-SNAPSHOT.jar + + + ext/dec-0.1.2.jar + release\modules\ext\dec-0.1.2.jar + ext/google-http-client-jackson2-1.29.0.jar release/modules/ext/google-http-client-jackson2-1.29.0.jar @@ -766,20 +766,24 @@ release/modules/ext/google-cloud-translate-1.70.0.jar - ext/gax-1.44.0.jar - release/modules/ext/gax-1.44.0.jar + ext/sevenzipjbinding-AllPlatforms.jar + release/modules/ext/sevenzipjbinding-AllPlatforms.jar ext/google-api-services-translate-v2-rev20170525-1.27.0.jar release/modules/ext/google-api-services-translate-v2-rev20170525-1.27.0.jar - ext/okhttp-2.7.5.jar - release/modules/ext/okhttp-2.7.5.jar + ext/webp-imageio-sejda-0.1.0.jar + release/modules/ext/webp-imageio-sejda-0.1.0.jar - ext/okio-1.6.0.jar - release/modules/ext/okio-1.6.0.jar + ext/bcmail-jdk15on-1.60.jar + release\modules\ext\bcmail-jdk15on-1.60.jar + + + ext/vorbis-java-tika-0.8.jar + release\modules\ext\vorbis-java-tika-0.8.jar diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiterUtils.java b/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiterUtils.java new file mode 100644 index 0000000000..3b2a201e07 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiterUtils.java @@ -0,0 +1,49 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule; + +import java.io.File; +import java.nio.file.Paths; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; + +/** + * Class for methods to check if access should be limited to a feature + * + */ +final class AccessLimiterUtils { + + private final static String MULTI_USER_ACCESS_FILE_NAME = "mualimit"; // NON-NLS + private final static String MULTI_USER_ACCESS_FILE_PATH = Paths.get(PlatformUtil.getUserConfigDirectory(), MULTI_USER_ACCESS_FILE_NAME).toString(); + + /** + * Check if privileges regarding multi-user cases should be restricted. + * + * @return True if privileges should be restricted, false otherwise. + */ + static boolean limitMultiUserAccess() { + return new File(MULTI_USER_ACCESS_FILE_PATH).exists(); + } + + /** + * Private constructor for a utility class + */ + private AccessLimiterUtils() { + //private constructer left empty intentionally + } +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED index 594b458990..79488c932f 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED @@ -117,7 +117,7 @@ CTL_CaseNewAction=New Case CTL_CaseDetailsAction=Case Details CTL_CaseDeleteAction=Delete Case CTL_CaseOpenAction=Open Case -CTL_UnpackagePortableCaseAction=Unpackage Portable Case +CTL_UnpackagePortableCaseAction=Unpack and Open Portable Case EditOptionalCasePropertiesPanel.cancelButton.text=Cancel EditOptionalCasePropertiesPanel.saveButton.text=Save GeneralFilter.encaseImageDesc.text=Encase Images (*.e01) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index f455a0c9fb..0a4586dd42 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1049,7 +1049,7 @@ public class Case { /* * Enable the case-specific actions. */ - CallableSystemAction.get(AddImageAction.class).setEnabled(true); + CallableSystemAction.get(AddImageAction.class).setEnabled(Case.getCurrentCase().getMetadata().getCaseType() == CaseType.SINGLE_USER_CASE || !AccessLimiterUtils.limitMultiUserAccess()); CallableSystemAction.get(CaseCloseAction.class).setEnabled(true); CallableSystemAction.get(CaseDetailsAction.class).setEnabled(true); CallableSystemAction.get(DataSourceSummaryAction.class).setEnabled(true); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java index aedbb3671f..789913d78a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java @@ -61,7 +61,7 @@ final class NewCaseVisualPanel1 extends JPanel implements DocumentListener { */ void readSettings() { caseNameTextField.setText(""); - if (UserPreferences.getIsMultiUserModeEnabled()) { + if (UserPreferences.getIsMultiUserModeEnabled() && !AccessLimiterUtils.limitMultiUserAccess()) { multiUserCaseRadioButton.setEnabled(true); multiUserCaseRadioButton.setSelected(true); } else { diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseAction.java b/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseAction.java index c17a6ba5c9..fe10350a58 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseAction.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseAction.java @@ -29,10 +29,10 @@ import org.openide.windows.WindowManager; @ActionID(category = "Case", id = "org.sleuthkit.autopsy.casemodule.UnpackagePortableCaseAction") @ActionRegistration(displayName = "#CTL_UnpackagePortableCaseAction", lazy = false) -@Messages({"CTL_UnpackagePortableCaseAction=Unpackage Portable Case"}) +@Messages({"CTL_UnpackagePortableCaseAction=Unpack and Open Portable Case"}) /** * Unpackage Portable Case action for the Case menu to allow the user to - * decompress a portable case. + * decompress a portable case and open it. */ public class UnpackagePortableCaseAction extends CallableSystemAction { diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseProgressDialog.java b/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseProgressDialog.java index 5f0f318183..656b376432 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseProgressDialog.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/UnpackagePortableCaseProgressDialog.java @@ -30,6 +30,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import javax.swing.JFrame; import javax.swing.SwingWorker; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; import org.openide.modules.InstalledFileLocator; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; @@ -189,7 +191,15 @@ class UnpackagePortableCaseProgressDialog extends javax.swing.JDialog implements setDisplayError(Bundle.UnpackageWorker_doInBackground_errorCompressingCase()); throw new TskCoreException("Error unpackaging case", ex); // NON-NLS } - + + try { + String caseFileDirectory = FilenameUtils.getBaseName(packagedCase); + String caseDirectory = StringUtils.substringBefore(caseFileDirectory, ".zip"); + Case.openAsCurrentCase(outputFolder + File.separator + caseDirectory + File.separator + caseDirectory + ".aut"); // NON-NLS + } catch (CaseActionException ex) { + throw new TskCoreException("Error opening case after unpacking it.", ex); // NON-NLS + } + success.set(true); return null; } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/Services.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/Services.java index 02f98f2887..81bc6a643c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/Services.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/Services.java @@ -80,7 +80,7 @@ public class Services implements Closeable { /** * Gets the artifacts blackboard for the current case. * - * @return @org.sleuthkit.datamodel.Blackboard Blackboard for the current + * @return org.sleuthkit.datamodel.Blackboard Blackboard for the current * case. */ public org.sleuthkit.datamodel.Blackboard getArtifactsBlackboard() { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index a98adf8b5c..d05f4a9635 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -1942,6 +1942,58 @@ abstract class AbstractSqlEamDb implements EamDb { return caseNames.stream().collect(Collectors.toList()); } + /** + * Gets list of distinct case display names, where each case has 1+ Artifact + * Instance matching eamArtifact. + * + * @param aType EamArtifact.Type to search for + * @param value Value to search for + * + * @return List of cases containing this artifact with instances marked as + * bad + * + * @throws EamDbException + */ + @Override + public List getListCasesHavingArtifactInstances(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { + + String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); + + Connection conn = connect(); + + Collection caseNames = new LinkedHashSet<>(); + + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); + String sql + = "SELECT DISTINCT case_name FROM " + + tableName + + " INNER JOIN cases ON " + + tableName + + ".case_id=cases.id WHERE " + + tableName + + ".value=? "; + + try { + preparedStatement = conn.prepareStatement(sql); + preparedStatement.setString(1, normalizedValue); + resultSet = preparedStatement.executeQuery(); + while (resultSet.next()) { + caseNames.add(resultSet.getString("case_name")); + } + } catch (SQLException ex) { + throw new EamDbException("Error getting notable artifact instances.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + + return caseNames.stream().collect(Collectors.toList()); + } + /** * Remove a reference set and all entries contained in it. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 98914afeec..18a21d0ab6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -211,8 +211,11 @@ public class EamArtifactUtil { TskData.FileKnown.UNKNOWN, bbSourceFile.getId()); - } catch (TskCoreException | EamDbException | CorrelationAttributeNormalizationException ex) { - logger.log(Level.SEVERE, "Error creating artifact instance.", ex); // NON-NLS + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error getting AbstractFile for artifact: " + bbArtifact.toString(), ex); // NON-NLS + return null; + } catch (EamDbException | CorrelationAttributeNormalizationException ex) { + logger.log(Level.WARNING, "Error creating artifact instance for artifact: " + bbArtifact.toString(), ex); // NON-NLS return null; } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Case is closed.", ex); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index 25ab69aeb9..c28c0537f7 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -475,6 +475,20 @@ public interface EamDb { */ List getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; + /** + * Gets list of distinct case display names, where each case has 1+ Artifact + * Instance matching eamArtifact. + * + * @param aType EamArtifact.Type to search for + * @param value Value to search for + * + * @return List of cases containing this artifact with instances marked as + * bad + * + * @throws EamDbException + */ + List getListCasesHavingArtifactInstances(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; + /** * Remove a reference set and all values contained in it. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 60c2ef98ab..282e225135 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -229,10 +229,13 @@ public class IngestEventsListener { "# {0} - typeName", "# {1} - count", "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"}) - static private void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact) { + static private void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List caseDisplayNames) { Collection attributesForNewArtifact = Arrays.asList(new BlackboardAttribute( TSK_SET_NAME, MODULE_NAME, Bundle.IngestEventsListener_prevExists_text()), + new BlackboardAttribute( + TSK_COMMENT, MODULE_NAME, + Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))), new BlackboardAttribute( TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, originalArtifact.getArtifactID())); @@ -482,9 +485,11 @@ public class IngestEventsListener { try { //only alert to previous instances when they were in another case List previousOccurences = dbManager.getArtifactInstancesByTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); + List caseDisplayNames; for (CorrelationAttributeInstance instance : previousOccurences) { if (!instance.getCorrelationCase().getCaseUUID().equals(eamArtifact.getCorrelationCase().getCaseUUID())) { - makeAndPostPreviousSeenArtifact(bbArtifact); + caseDisplayNames = dbManager.getListCasesHavingArtifactInstances(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); + makeAndPostPreviousSeenArtifact(bbArtifact, caseDisplayNames); break; } } diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties index 8675363370..9c4f680119 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties @@ -2,16 +2,11 @@ OpenIDE-Module-Name=CommandLineAutopsy OptionsCategory_Keywords_Command_Line_Ingest_Settings=Command Line Ingest Settings OptionsCategory_Keywords_General=Options OptionsCategory_Name_Command_Line_Ingest=Command Line Ingest -CommandLineIngestSettingsPanel.ResultsDirectoryUnspecified=Output folder must be set -CommandLineIngestSettingsPanel.PathInvalid=Path is not valid -CommandLineIngestSettingsPanel.CannotAccess=Cannot access -CommandLineIngestSettingsPanel.CheckPermissions=Check permissions. -CommandLineIngestSettingsPanel.jLabelSelectOutputFolder.text=Select output folder: -CommandLineIngestSettingsPanel.jLabelInvalidResultsFolder.text=jLabelInvalidOutputFolder -CommandLineIngestSettingsPanel.outputPathTextField.toolTipText=Output folder for command line processing, i.e., the location where case folder will be created by command line processing mode. -CommandLineIngestSettingsPanel.outputPathTextField.text= -CommandLineIngestSettingsPanel.browseOutputFolderButton.text=Browse -CommandLineIngestSettingsPanel.bnEditIngestSettings.toolTipText=Ingest job settings for the command line processing mode context. -CommandLineIngestSettingsPanel.bnEditIngestSettings.text=Ingest Module Settings CommandLinePanel.jLabel1.text=Ingest is running from command line CommandLineStartupWindow.title.text=Running in Command Line Mode +CommandLineIngestSettingsPanel.bnEditIngestSettings.text=Configure Ingest +CommandLineIngestSettingsPanel.bnEditReportSettings.actionCommand=Report Module Settings +CommandLineIngestSettingsPanel.bnEditReportSettings.toolTipText=Report generation settings for the command line processing mode context. +CommandLineIngestSettingsPanel.bnEditReportSettings.text=Configure Reporting +CommandLineIngestSettingsPanel.jLabelDescription.text=You can create cases, add data sources, run ingest modules, and generate reports from the command line.
This options panel allows you to configure the settings to use when running ingest modules and generating reports.
See the user documentation for details. +CommandLineIngestSettingsPanel.bnEditIngestSettings.toolTipText=Ingest job settings for the command line processing mode context. diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED index 8675363370..9c4f680119 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED @@ -2,16 +2,11 @@ OpenIDE-Module-Name=CommandLineAutopsy OptionsCategory_Keywords_Command_Line_Ingest_Settings=Command Line Ingest Settings OptionsCategory_Keywords_General=Options OptionsCategory_Name_Command_Line_Ingest=Command Line Ingest -CommandLineIngestSettingsPanel.ResultsDirectoryUnspecified=Output folder must be set -CommandLineIngestSettingsPanel.PathInvalid=Path is not valid -CommandLineIngestSettingsPanel.CannotAccess=Cannot access -CommandLineIngestSettingsPanel.CheckPermissions=Check permissions. -CommandLineIngestSettingsPanel.jLabelSelectOutputFolder.text=Select output folder: -CommandLineIngestSettingsPanel.jLabelInvalidResultsFolder.text=jLabelInvalidOutputFolder -CommandLineIngestSettingsPanel.outputPathTextField.toolTipText=Output folder for command line processing, i.e., the location where case folder will be created by command line processing mode. -CommandLineIngestSettingsPanel.outputPathTextField.text= -CommandLineIngestSettingsPanel.browseOutputFolderButton.text=Browse -CommandLineIngestSettingsPanel.bnEditIngestSettings.toolTipText=Ingest job settings for the command line processing mode context. -CommandLineIngestSettingsPanel.bnEditIngestSettings.text=Ingest Module Settings CommandLinePanel.jLabel1.text=Ingest is running from command line CommandLineStartupWindow.title.text=Running in Command Line Mode +CommandLineIngestSettingsPanel.bnEditIngestSettings.text=Configure Ingest +CommandLineIngestSettingsPanel.bnEditReportSettings.actionCommand=Report Module Settings +CommandLineIngestSettingsPanel.bnEditReportSettings.toolTipText=Report generation settings for the command line processing mode context. +CommandLineIngestSettingsPanel.bnEditReportSettings.text=Configure Reporting +CommandLineIngestSettingsPanel.jLabelDescription.text=You can create cases, add data sources, run ingest modules, and generate reports from the command line.
This options panel allows you to configure the settings to use when running ingest modules and generating reports.
See the user documentation for details. +CommandLineIngestSettingsPanel.bnEditIngestSettings.toolTipText=Ingest job settings for the command line processing mode context. diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java new file mode 100755 index 0000000000..8888ac72de --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java @@ -0,0 +1,82 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commandlineingest; + +import java.util.HashMap; +import java.util.Map; + +/** + * Class that contains list of input parameters passed in via command line. + */ +class CommandLineCommand { + + /** + * An enumeration of command types. + */ + static enum CommandType { + CREATE_CASE, + ADD_DATA_SOURCE, + RUN_INGEST, + LIST_ALL_DATA_SOURCES, + GENERATE_REPORTS; + } + + /** + * An enumeration of input types. + */ + static enum InputType { + CASE_NAME, + CASES_BASE_DIR_PATH, + CASE_FOLDER_PATH, + DATA_SOURCE_PATH, + DATA_SOURCE_ID, + INGEST_PROFILE_NAME; + } + + private final CommandType type; + private final Map inputs = new HashMap<>(); + + CommandLineCommand(CommandType type) { + this.type = type; + } + + /** + * Adds an parameter to the command. + * + * @param inputName Input parameter name + * @param inputValue Input parameter value + */ + void addInputValue(String inputName, String inputValue) { + inputs.put(inputName, inputValue); + } + + /** + * @return the inputs + */ + Map getInputs() { + return inputs; + } + + /** + * @return the type + */ + CommandType getType() { + return type; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java index 29faa16356..4d04e5dbbf 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java @@ -24,11 +24,14 @@ import java.io.File; import java.io.FilenameFilter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.Collection; import java.util.EnumSet; import java.util.Iterator; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import org.netbeans.spi.sendopts.OptionProcessor; @@ -38,9 +41,11 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CaseActionException; import org.sleuthkit.autopsy.casemodule.CaseDetails; import org.sleuthkit.autopsy.casemodule.CaseMetadata; +import static org.sleuthkit.autopsy.casemodule.CaseMetadata.getFileExtension; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; +import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.NO_ERRORS; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.TimeStampUtils; @@ -54,21 +59,27 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; -import org.sleuthkit.autopsy.report.ReportProgressPanel; -import org.sleuthkit.autopsy.report.caseuco.CaseUcoFormatExporter; -import org.sleuthkit.autopsy.report.caseuco.ReportCaseUco; +import org.sleuthkit.autopsy.ingest.IngestProfiles; +import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; +import org.sleuthkit.autopsy.modules.interestingitems.FilesSetsManager; +import org.sleuthkit.autopsy.progress.LoggingProgressIndicator; +import org.sleuthkit.autopsy.report.infrastructure.ReportGenerator; +import org.sleuthkit.autopsy.report.infrastructure.ReportProgressIndicator; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; /** - * Allows Autopsy to be invoked with a command line arguments. Causes Autopsy to - * create a case, add a specified data source, run ingest on that data source, - * produce a CASE/UCO report and exit. + * Allows Autopsy to be invoked with command line arguments. Arguments exist to + * cause Autopsy to create a case, add a specified data source, run ingest on + * that data source, list all data sources in the case, and generate reports. */ public class CommandLineIngestManager { private static final Logger LOGGER = Logger.getLogger(CommandLineIngestManager.class.getName()); private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.CANCELLED, IngestManager.IngestJobEvent.COMPLETED); - private Path rootOutputDirectory; + private Case caseForJob = null; + private AutoIngestDataSource dataSource = null; + private static final String LOG_DIR_NAME = "Command Output"; public CommandLineIngestManager() { } @@ -103,6 +114,10 @@ public class CommandLineIngestManager { } } + /** + * Requests the list of command line commands from command line options + * processor and executes the commands one by one. + */ @Override public void run() { LOGGER.log(Level.INFO, "Job processing task started"); @@ -110,8 +125,7 @@ public class CommandLineIngestManager { try { // read command line inputs LOGGER.log(Level.INFO, "Autopsy is running from command line"); //NON-NLS - String dataSourcePath = ""; - String baseCaseName = ""; + List commands = null; // first look up all OptionProcessors and get input data from CommandLineOptionProcessor Collection optionProcessors = Lookup.getDefault().lookupAll(OptionProcessor.class); @@ -120,86 +134,173 @@ public class CommandLineIngestManager { // find CommandLineOptionProcessor OptionProcessor processor = optionsIterator.next(); if (processor instanceof CommandLineOptionProcessor) { - // check if we are running from command line - dataSourcePath = ((CommandLineOptionProcessor) processor).getPathToDataSource(); - baseCaseName = ((CommandLineOptionProcessor) processor).getBaseCaseName(); + // check if we are running from command line + commands = ((CommandLineOptionProcessor) processor).getCommands(); } } - LOGGER.log(Level.INFO, "Data source path = {0}", dataSourcePath); //NON-NLS - LOGGER.log(Level.INFO, "Case name = {0}", baseCaseName); //NON-NLS - System.out.println("Data source path = " + dataSourcePath); - System.out.println("Case name = " + baseCaseName); - - // verify inputs - if (dataSourcePath.isEmpty()) { - LOGGER.log(Level.SEVERE, "Data source path not specified"); - System.out.println("Data source path not specified"); + if (commands == null || commands.isEmpty()) { + LOGGER.log(Level.SEVERE, "No command line commands specified"); + System.err.println("No command line commands specified"); return; } - if (baseCaseName.isEmpty()) { - LOGGER.log(Level.SEVERE, "Case name not specified"); - System.out.println("Case name not specified"); - return; - } - - if (!(new File(dataSourcePath).exists())) { - LOGGER.log(Level.SEVERE, "Data source file not found {0}", dataSourcePath); - System.out.println("Data source file not found " + dataSourcePath); - return; - } - - // read options panel configuration - String rootOutputDir = UserPreferences.getCommandLineModeResultsFolder(); - LOGGER.log(Level.INFO, "Output directory = {0}", rootOutputDir); //NON-NLS - System.out.println("Output directory = " + rootOutputDir); - - if (rootOutputDir.isEmpty()) { - LOGGER.log(Level.SEVERE, "Output directory not specified, please configure Command Line Options Panel (in Tools -> Options)"); - System.out.println("Output directory not specified, please configure Command Line Options Panel (in Tools -> Options)"); - return; - } - - if (!(new File(rootOutputDir).exists())) { - LOGGER.log(Level.SEVERE, "The output directory doesn't exist {0}", rootOutputDir); - System.out.println("The output directory doesn't exist " + rootOutputDir); - return; - } - rootOutputDirectory = Paths.get(rootOutputDir); - - // open case - Case caseForJob; try { - caseForJob = openCase(baseCaseName); - } catch (CaseActionException ex) { - LOGGER.log(Level.SEVERE, "Error creating or opening case " + baseCaseName, ex); - System.out.println("Error creating or opening case " + baseCaseName); - return; - } + // Commands are already stored in order in which they should be executed + for (CommandLineCommand command : commands) { + CommandLineCommand.CommandType type = command.getType(); + switch (type) { + case CREATE_CASE: + try { + LOGGER.log(Level.INFO, "Processing 'Create Case' command"); + System.out.println("Processing 'Create Case' command"); + Map inputs = command.getInputs(); + String baseCaseName = inputs.get(CommandLineCommand.InputType.CASE_NAME.name()); + String rootOutputDirectory = inputs.get(CommandLineCommand.InputType.CASES_BASE_DIR_PATH.name()); + openCase(baseCaseName, rootOutputDirectory); - if (caseForJob == null) { - LOGGER.log(Level.SEVERE, "Error creating or opening case {0}", baseCaseName); - System.out.println("Error creating or opening case " + baseCaseName); - return; - } + String outputDirPath = getOutputDirPath(caseForJob); + OutputGenerator.saveCreateCaseOutput(caseForJob, outputDirPath, baseCaseName); + } catch (CaseActionException ex) { + String baseCaseName = command.getInputs().get(CommandLineCommand.InputType.CASE_NAME.name()); + LOGGER.log(Level.SEVERE, "Error creating or opening case " + baseCaseName, ex); + System.err.println("Error creating or opening case " + baseCaseName); + // Do not process any other commands + return; + } + break; + case ADD_DATA_SOURCE: + try { + LOGGER.log(Level.INFO, "Processing 'Add Data Source' command"); + System.out.println("Processing 'Add Data Source' command"); + Map inputs = command.getInputs(); - AutoIngestDataSource dataSource = new AutoIngestDataSource("", Paths.get(dataSourcePath)); - try { - // run data source processor - runDataSourceProcessor(caseForJob, dataSource); + // open the case, if it hasn't been already opened by CREATE_CASE command + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + openCase(caseDirPath); + } - // run ingest manager - analyze(dataSource); + String dataSourcePath = inputs.get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); + dataSource = new AutoIngestDataSource("", Paths.get(dataSourcePath)); + runDataSourceProcessor(caseForJob, dataSource); - // generate CASE-UCO report - Long selectedDataSourceId = getDataSourceId(dataSource); - Path reportFolderPath = Paths.get(caseForJob.getReportDirectory(), "CASE-UCO", "Data_Source_ID_" + selectedDataSourceId.toString() + "_" + TimeStampUtils.createTimeStamp(), ReportCaseUco.getReportFileName()); //NON_NLS - ReportProgressPanel progressPanel = new ReportProgressPanel("CASE_UCO", rootOutputDir); // dummy progress panel - CaseUcoFormatExporter.generateReport(selectedDataSourceId, reportFolderPath.toString(), progressPanel); - } catch (InterruptedException | AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException | AnalysisStartupException ex) { - LOGGER.log(Level.SEVERE, "Unable to ingest data source " + dataSourcePath + ". Exiting...", ex); - System.out.println("Unable to ingest data source " + dataSourcePath + ". Exiting..."); + String outputDirPath = getOutputDirPath(caseForJob); + OutputGenerator.saveAddDataSourceOutput(caseForJob, dataSource, outputDirPath); + } catch (InterruptedException | AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException | CaseActionException ex) { + String dataSourcePath = command.getInputs().get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); + LOGGER.log(Level.SEVERE, "Error adding data source " + dataSourcePath, ex); + System.err.println("Error adding data source " + dataSourcePath); + // Do not process any other commands + return; + } + break; + case RUN_INGEST: + try { + LOGGER.log(Level.INFO, "Processing 'Run Ingest' command"); + System.out.println("Processing 'Run Ingest' command"); + Map inputs = command.getInputs(); + + // open the case, if it hasn't been already opened by CREATE_CASE or ADD_DATA_SOURCE commands + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + openCase(caseDirPath); + } + + // populate the AutoIngestDataSource structure, if that hasn't been done by ADD_DATA_SOURCE command + if (dataSource == null) { + + String dataSourceId = inputs.get(CommandLineCommand.InputType.DATA_SOURCE_ID.name()); + Long dataSourceObjId = Long.valueOf(dataSourceId); + + // get Content object for the data source + Content content = null; + try { + content = Case.getCurrentCaseThrows().getSleuthkitCase().getContentById(dataSourceObjId); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Exception while trying to find data source with object ID " + dataSourceId, ex); + System.err.println("Exception while trying to find data source with object ID " + dataSourceId); + // Do not process any other commands + return; + } + + if (content == null) { + LOGGER.log(Level.SEVERE, "Unable to find data source with object ID {0}", dataSourceId); + System.out.println("Unable to find data source with object ID " + dataSourceId); + // Do not process any other commands + return; + } + + // populate the AutoIngestDataSource structure + dataSource = new AutoIngestDataSource("", Paths.get(content.getName())); + List contentList = Arrays.asList(new Content[]{content}); + List errorList = new ArrayList<>(); + dataSource.setDataSourceProcessorOutput(NO_ERRORS, errorList, contentList); + } + + // run ingest + String ingestProfile = inputs.get(CommandLineCommand.InputType.INGEST_PROFILE_NAME.name()); + analyze(dataSource, ingestProfile); + } catch (InterruptedException | CaseActionException ex) { + String dataSourcePath = command.getInputs().get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); + LOGGER.log(Level.SEVERE, "Error running ingest on data source " + dataSourcePath, ex); + System.err.println("Error running ingest on data source " + dataSourcePath); + // Do not process any other commands + return; + } + break; + + case LIST_ALL_DATA_SOURCES: + try { + LOGGER.log(Level.INFO, "Processing 'List All Data Sources' command"); + System.out.println("Processing 'List All Data Sources' command"); + Map inputs = command.getInputs(); + + // open the case, if it hasn't been already opened by previous command + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + openCase(caseDirPath); + } + + String outputDirPath = getOutputDirPath(caseForJob); + OutputGenerator.listAllDataSources(caseForJob, outputDirPath); + } catch (CaseActionException ex) { + String caseDirPath = command.getInputs().get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + LOGGER.log(Level.SEVERE, "Error opening case in case directory: " + caseDirPath, ex); + System.err.println("Error opening case in case directory: " + caseDirPath); + // Do not process any other commands + return; + } + break; + + case GENERATE_REPORTS: + try { + LOGGER.log(Level.INFO, "Processing 'Generate Reports' command"); + System.out.println("Processing 'Generate Reports' command"); + Map inputs = command.getInputs(); + + // open the case, if it hasn't been already opened by previous command + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + openCase(caseDirPath); + } + + // generate reports + ReportProgressIndicator progressIndicator = new ReportProgressIndicator(new LoggingProgressIndicator()); + ReportGenerator generator = new ReportGenerator(CommandLineIngestSettingsPanel.getReportingConfigName(), progressIndicator); + generator.generateReports(); + } catch (CaseActionException ex) { + String caseDirPath = command.getInputs().get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + LOGGER.log(Level.SEVERE, "Error opening case in case directory: " + caseDirPath, ex); + System.err.println("Error opening case in case directory: " + caseDirPath); + // Do not process any other commands + return; + } + break; + default: + break; + } + } } catch (Throwable ex) { /* * Unexpected runtime exceptions firewall. This task is @@ -208,15 +309,15 @@ public class CommandLineIngestManager { * Future, so this ensures that such errors get * logged. */ - LOGGER.log(Level.SEVERE, "Unexpected error while ingesting data source " + dataSourcePath, ex); - System.out.println("Unexpected error while ingesting data source " + dataSourcePath + ". Exiting..."); + LOGGER.log(Level.SEVERE, "Unexpected error", ex); + System.err.println("Unexpected error. Exiting..."); } finally { try { Case.closeCurrentCase(); } catch (CaseActionException ex) { LOGGER.log(Level.WARNING, "Exception while closing case", ex); - System.out.println("Exception while closing case"); + System.err.println("Exception while closing case"); } } @@ -230,29 +331,25 @@ public class CommandLineIngestManager { } /** - * Provides object ID of the data source by reading it from Content - * object. + * Creates a new case using arguments passed in from command line + * CREATE_CASE command. * - * @param dataSource DataSource object + * @param baseCaseName Case name + * @param rootOutputDirectory Full path to directory in which case + * output folder will be created * - * @return object ID + * @throws CaseActionException */ - private Long getDataSourceId(AutoIngestDataSource dataSource) { - Content content = dataSource.getContent().get(0); - return content.getId(); - } + private void openCase(String baseCaseName, String rootOutputDirectory) throws CaseActionException { - private Case openCase(String baseCaseName) throws CaseActionException { - - LOGGER.log(Level.INFO, "Opening case {0}", baseCaseName); - - Path caseDirectoryPath = findCaseDirectory(rootOutputDirectory, baseCaseName); + LOGGER.log(Level.INFO, "Opening case {0} in directory {1}", new Object[]{baseCaseName, rootOutputDirectory}); + Path caseDirectoryPath = findCaseDirectory(Paths.get(rootOutputDirectory), baseCaseName); if (null != caseDirectoryPath) { // found an existing case directory for same case name. the input case name must be unique. Exit. LOGGER.log(Level.SEVERE, "Case {0} already exists. Case name must be unique. Exiting", baseCaseName); throw new CaseActionException("Case " + baseCaseName + " already exists. Case name must be unique. Exiting"); } else { - caseDirectoryPath = createCaseFolderPath(rootOutputDirectory, baseCaseName); + caseDirectoryPath = createCaseFolderPath(Paths.get(rootOutputDirectory), baseCaseName); // Create the case directory Case.createCaseDirectory(caseDirectoryPath.toString(), Case.CaseType.SINGLE_USER_CASE); @@ -261,9 +358,57 @@ public class CommandLineIngestManager { Case.createAsCurrentCase(Case.CaseType.SINGLE_USER_CASE, caseDirectoryPath.toString(), caseDetails); } - Case caseForJob = Case.getCurrentCase(); + caseForJob = Case.getCurrentCase(); LOGGER.log(Level.INFO, "Opened case {0}", caseForJob.getName()); - return caseForJob; + } + + /** + * Opens existing case. + * + * @param caseFolderPath full path to case directory + * + * @throws CaseActionException + */ + private void openCase(String caseFolderPath) throws CaseActionException { + + LOGGER.log(Level.INFO, "Opening case in directory {0}", caseFolderPath); + + String metadataFilePath = findAutFile(caseFolderPath); + Case.openAsCurrentCase(metadataFilePath); + + caseForJob = Case.getCurrentCase(); + LOGGER.log(Level.INFO, "Opened case {0}", caseForJob.getName()); + } + + /** + * Finds the path to the .aut file for the specified case directory. + * + * @param caseDirectory the directory to check for a .aut file + * + * @return the path to the first .aut file found in the directory + * + * @throws CaseActionException if there was an issue finding a .aut file + */ + private String findAutFile(String caseDirectory) throws CaseActionException { + File caseFolder = Paths.get(caseDirectory).toFile(); + if (caseFolder.exists()) { + /* + * Search for '*.aut' files. + */ + File[] fileArray = caseFolder.listFiles(); + if (fileArray == null) { + throw new CaseActionException("No files found in case directory"); + } + String autFilePath = null; + for (File file : fileArray) { + String name = file.getName().toLowerCase(); + if (autFilePath == null && name.endsWith(getFileExtension())) { + return file.getAbsolutePath(); + } + } + throw new CaseActionException("No .aut files found in case directory"); + } + throw new CaseActionException("Case directory was not found"); } /** @@ -274,11 +419,16 @@ public class CommandLineIngestManager { * @param dataSource The data source. * * @throws - * AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException if - * there was a DSP processing error + * AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorExceptioif + * there + * was + * a + * DSP + * processing + * error * - * @throws InterruptedException if the thread running the job processing - * task is interrupted while blocked, i.e., if auto ingest is shutting down. + * @throws ead running the job processing task is interrupted while + * blocked, i.e., if auto ingest is shutting down. */ private void runDataSourceProcessor(Case caseForJob, AutoIngestDataSource dataSource) throws InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { @@ -375,24 +525,60 @@ public class CommandLineIngestManager { /** * Analyzes the data source content returned by the data source * processor using the configured set of data source level and file - * level analysis modules. + * level analysis modules. If an ingest profile is specified, load that + * profile (profile = ingest context + ingest filter) for ingest. + * Otherwise use baseline configuration. * - * @param dataSource The data source to analyze. + * @param dataSource The data source to analyze. + * @param ingestProfileName Name of ingest profile to use (optional) * * @throws AnalysisStartupException if there is an error analyzing the - * data source. - * @throws InterruptedException if the thread running the job processing - * task is interrupted while blocked, i.e., if auto ingest is shutting - * down. + * data source. + * @throws InterruptedException if the thread running the job + * processing task is interrupted while + * blocked, i.e., if auto ingest is + * shutting down. */ - private void analyze(AutoIngestDataSource dataSource) throws AnalysisStartupException, InterruptedException { + private void analyze(AutoIngestDataSource dataSource, String ingestProfileName) throws AnalysisStartupException, InterruptedException { LOGGER.log(Level.INFO, "Starting ingest modules analysis for {0} ", dataSource.getPath()); + + // configure ingest profile and file filter + IngestProfiles.IngestProfile selectedProfile = null; + FilesSet selectedFileSet = null; + if (!ingestProfileName.isEmpty()) { + selectedProfile = getSelectedProfile(ingestProfileName); + if (selectedProfile == null) { + // unable to find the user specified profile + LOGGER.log(Level.SEVERE, "Unable to find ingest profile: {0}. Ingest cancelled!", ingestProfileName); + System.err.println("Unable to find ingest profile: " + ingestProfileName + ". Ingest cancelled!"); + return; + } + + // get FileSet filter associated with this profile + selectedFileSet = getSelectedFilter(selectedProfile.getFileIngestFilter()); + if (selectedFileSet == null) { + // unable to find the user specified profile + LOGGER.log(Level.SEVERE, "Unable to find file filter {0} for ingest profile: {1}. Ingest cancelled!", new Object[]{selectedProfile.getFileIngestFilter(), ingestProfileName}); + System.err.println("Unable to find file filter " + selectedProfile.getFileIngestFilter() + " for ingest profile: " + ingestProfileName + ". Ingest cancelled!"); + return; + } + } + IngestJobEventListener ingestJobEventListener = new IngestJobEventListener(); IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { - IngestJobSettings ingestJobSettings = new IngestJobSettings(UserPreferences.getCommandLineModeIngestModuleContextString()); + IngestJobSettings ingestJobSettings; + if (selectedProfile == null || selectedFileSet == null) { + // use baseline configuration + ingestJobSettings = new IngestJobSettings(UserPreferences.getCommandLineModeIngestModuleContextString()); + } else { + // load the custom ingest + ingestJobSettings = new IngestJobSettings(selectedProfile.toString()); + ingestJobSettings.setFileFilter(selectedFileSet); + } + List settingsWarnings = ingestJobSettings.getWarnings(); if (settingsWarnings.isEmpty()) { IngestJobStartResult ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); @@ -447,6 +633,50 @@ public class CommandLineIngestManager { } } + /** + * Gets the specified ingest profile from the list of all existing + * ingest profiles. + * + * @param ingestProfileName Ingest profile name + * + * @return IngestProfile object, or NULL if the profile doesn't exist + */ + private IngestProfiles.IngestProfile getSelectedProfile(String ingestProfileName) { + + IngestProfiles.IngestProfile selectedProfile = null; + // lookup the profile by name + for (IngestProfiles.IngestProfile profile : IngestProfiles.getIngestProfiles()) { + if (profile.toString().equalsIgnoreCase(ingestProfileName)) { + // found the profile + selectedProfile = profile; + break; + } + } + return selectedProfile; + } + + /** + * Gets the specified file filter from the list of all existing file + * filters (custom and standard). + * + * @param filterName Name of the file filter + * + * @return FilesSet object, or NULL if the filter doesn't exist + */ + private FilesSet getSelectedFilter(String filterName) { + try { + Map fileIngestFilters = FilesSetsManager.getInstance() + .getCustomFileIngestFilters(); + for (FilesSet fSet : FilesSetsManager.getStandardFileIngestFilters()) { + fileIngestFilters.put(fSet.getName(), fSet); + } + return fileIngestFilters.get(filterName); + } catch (FilesSetsManager.FilesSetsManagerException ex) { + LOGGER.log(Level.SEVERE, "Failed to get file ingest filter: " + filterName, ex); //NON-NLS + return null; + } + } + /** * Creates a case folder path. Does not create the folder described by * the path. @@ -456,7 +686,7 @@ public class CommandLineIngestManager { * * @return A case folder path with a time stamp suffix. */ - Path createCaseFolderPath(Path caseFoldersPath, String caseName) { + private Path createCaseFolderPath(Path caseFoldersPath, String caseName) { String folderName = caseName + "_" + TimeStampUtils.createTimeStamp(); return Paths.get(caseFoldersPath.toString(), folderName); } @@ -471,7 +701,7 @@ public class CommandLineIngestManager { * * @return The path of the case folder, or null if it is not found. */ - Path findCaseDirectory(Path folderToSearch, String caseName) { + private Path findCaseDirectory(Path folderToSearch, String caseName) { File searchFolder = new File(folderToSearch.toString()); if (!searchFolder.isDirectory()) { return null; @@ -489,6 +719,17 @@ public class CommandLineIngestManager { return caseFolderPath; } + /** + * Returns full path to directory where command outputs should be saved. + * + * @param caseForJob Case object + * + * @return Full path to directory where command outputs should be saved + */ + private String getOutputDirPath(Case caseForJob) { + return caseForJob.getCaseDirectory() + File.separator + LOG_DIR_NAME; + } + /** * An ingest job event listener that allows the job processing task to * block until the analysis of a data source by the data source level diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.form index 2333564f89..c20e802081 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.form @@ -1,6 +1,11 @@
+ + + + + @@ -16,7 +21,7 @@ - + @@ -31,6 +36,9 @@ + + + @@ -40,46 +48,35 @@ + + + - - + + - - - - - - - - - - + + - + - - - - - - - - - - - + + + - + + + @@ -88,10 +85,10 @@ - + - + @@ -101,58 +98,28 @@ - + - + - - - - - - - - - + - + - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java index 5d3a0787ce..63bad0bd23 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java @@ -20,20 +20,13 @@ package org.sleuthkit.autopsy.commandlineingest; import java.awt.BorderLayout; import java.awt.Cursor; -import java.io.File; -import java.nio.file.Files; import java.util.List; -import javax.swing.JFileChooser; import javax.swing.JOptionPane; -import javax.swing.event.DocumentEvent; -import javax.swing.event.DocumentListener; -import org.openide.util.NbBundle; import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobSettingsPanel; -import java.nio.file.Paths; -import org.sleuthkit.autopsy.coreutils.FileUtil; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.coreutils.Logger; +import static org.sleuthkit.autopsy.report.infrastructure.ReportWizardAction.doReportWizard; /** * Configuration panel for auto ingest settings. @@ -41,10 +34,12 @@ import org.sleuthkit.autopsy.coreutils.Logger; @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { - private final CommandLineIngestSettingsPanelController controller; - private final JFileChooser fc = new JFileChooser(); private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(CommandLineIngestSettingsPanel.class.getName()); + + private static final String REPORTING_CONFIGURATION_NAME = "CommandLineIngest"; + private static final boolean DISPLAY_CASE_SPECIFIC_DATA = false; // do not try to display case specific data + private static final boolean RUN_REPORTS = false; // do not generate reports as part of running the report wizard /** * Creates new form AutoIngestSettingsPanel @@ -52,171 +47,14 @@ public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { * @param theController Controller to notify of changes. */ public CommandLineIngestSettingsPanel(CommandLineIngestSettingsPanelController theController) { - controller = theController; initComponents(); - - load(true); - outputPathTextField.getDocument().addDocumentListener(new MyDocumentListener()); - jLabelInvalidResultsFolder.setText(""); } - - private class MyDocumentListener implements DocumentListener { - - @Override - public void changedUpdate(DocumentEvent e) { - valid(); - controller.changed(); - } - - @Override - public void removeUpdate(DocumentEvent e) { - valid(); - controller.changed(); - } - - @Override - public void insertUpdate(DocumentEvent e) { - valid(); - controller.changed(); - } - }; - + /** - * Load mode from persistent storage. - * - * @param inStartup True if we're doing the initial population of the UI + * @return the REPORTING_CONFIGURATION_NAME */ - final void load(boolean inStartup) { - - String results = org.sleuthkit.autopsy.commandlineingest.UserPreferences.getCommandLineModeResultsFolder(); - if (results != null) { - outputPathTextField.setText(results); - } else { - outputPathTextField.setText(""); - } - - valid(); - } - - /** - * Save mode to persistent storage. - */ - void store() { - String resultsFolderPath = getNormalizedFolderPath(outputPathTextField.getText().trim()); - org.sleuthkit.autopsy.commandlineingest.UserPreferences.setCommandLineModeResultsFolder(resultsFolderPath); - } - - /** - * Validate current panel settings. - */ - boolean valid() { - - if (validateResultsPath()) { - return true; - } - return false; - } - - /** - * Normalizes a path to make sure there are no "space" characters at the end - * - * @param path Path to a directory - * - * @return Path without "space" characters at the end - */ - String normalizePath(String path) { - - while (path.length() > 0) { - if (path.charAt(path.length() - 1) == ' ') { - path = path.substring(0, path.length() - 1); - } else { - break; - } - } - return path; - } - - /** - * Validates that a path is valid and points to a folder. - * - * @param path A path to be validated - * - * @return boolean returns true if valid and points to a folder, false - * otherwise - */ - boolean isFolderPathValid(String path) { - try { - File file = new File(normalizePath(path)); - - // check if it's a symbolic link - if (Files.isSymbolicLink(file.toPath())) { - return true; - } - - // local folder - if (file.exists() && file.isDirectory()) { - return true; - } - } catch (Exception ex) { - // Files.isSymbolicLink (and other "files" methods) throw exceptions on seemingly innocent inputs. - // For example, it will throw an exception when either " " is last character in path or - // a path starting with ":". - // We can just ignore these exceptions as they occur in process of user typing in the path. - return false; - } - return false; - } - - /** - * Returns a path that was normalized by file system. - * - * @param path A path to be normalized. Normalization occurs inside a call - * to new File(). - * - * @return String returns normalized OS path - */ - String getNormalizedFolderPath(String path) { - // removes "/", "\", and " " characters at the end of path string. - // normalizePath() removes spaces at the end of path and a call to "new File()" - // internally formats the path string to remove "/" and "\" characters at the end of path. - File file = new File(normalizePath(path)); - return file.getPath(); - } - - /** - * Validate results path. Display warnings if invalid. - */ - boolean validateResultsPath() { - - String outputPath = outputPathTextField.getText().trim(); - - if (outputPath.isEmpty()) { - jLabelInvalidResultsFolder.setVisible(true); - jLabelInvalidResultsFolder.setText(NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.ResultsDirectoryUnspecified")); - /* - NOTE: JIRA-4850: Returning false disables OK and Apply buttons for the entire - Tools->Options bar until the path is set. It was decided to only validate - the path if the path is set. - */ - return true; - } - - if (!isFolderPathValid(outputPath)) { - jLabelInvalidResultsFolder.setVisible(true); - jLabelInvalidResultsFolder.setText(NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.PathInvalid")); - return false; - } - - if (false == permissionsAppropriate(outputPath)) { - jLabelInvalidResultsFolder.setVisible(true); - jLabelInvalidResultsFolder.setText(NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.CannotAccess") - + " " + outputPath + " " - + NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.CheckPermissions")); - return false; - } - - jLabelInvalidResultsFolder.setVisible(false); - return true; + public static String getReportingConfigName() { + return REPORTING_CONFIGURATION_NAME; } private void displayIngestJobSettingsPanel() { @@ -261,14 +99,16 @@ public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { nodeScrollPane = new javax.swing.JScrollPane(); nodePanel = new javax.swing.JPanel(); bnEditIngestSettings = new javax.swing.JButton(); - browseOutputFolderButton = new javax.swing.JButton(); - outputPathTextField = new javax.swing.JTextField(); - jLabelInvalidResultsFolder = new javax.swing.JLabel(); - jLabelSelectOutputFolder = new javax.swing.JLabel(); + jLabelDescription = new javax.swing.JLabel(); + bnEditReportSettings = new javax.swing.JButton(); + + setPreferredSize(new java.awt.Dimension(810, 422)); nodeScrollPane.setMinimumSize(new java.awt.Dimension(0, 0)); + nodeScrollPane.setPreferredSize(new java.awt.Dimension(803, 553)); nodePanel.setMinimumSize(new java.awt.Dimension(100, 100)); + nodePanel.setPreferredSize(new java.awt.Dimension(801, 551)); org.openide.awt.Mnemonics.setLocalizedText(bnEditIngestSettings, org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.bnEditIngestSettings.text")); // NOI18N bnEditIngestSettings.setToolTipText(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.bnEditIngestSettings.toolTipText")); // NOI18N @@ -279,22 +119,17 @@ public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { } }); - org.openide.awt.Mnemonics.setLocalizedText(browseOutputFolderButton, org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.browseOutputFolderButton.text")); // NOI18N - browseOutputFolderButton.addActionListener(new java.awt.event.ActionListener() { + org.openide.awt.Mnemonics.setLocalizedText(jLabelDescription, org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.jLabelDescription.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(bnEditReportSettings, org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.bnEditReportSettings.text")); // NOI18N + bnEditReportSettings.setToolTipText(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.bnEditReportSettings.toolTipText")); // NOI18N + bnEditReportSettings.setActionCommand(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.bnEditReportSettings.actionCommand")); // NOI18N + bnEditReportSettings.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { - browseOutputFolderButtonActionPerformed(evt); + bnEditReportSettingsActionPerformed(evt); } }); - outputPathTextField.setText(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.outputPathTextField.text")); // NOI18N - outputPathTextField.setToolTipText(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.outputPathTextField.toolTipText")); // NOI18N - - jLabelInvalidResultsFolder.setForeground(new java.awt.Color(255, 0, 0)); - org.openide.awt.Mnemonics.setLocalizedText(jLabelInvalidResultsFolder, org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.jLabelInvalidResultsFolder.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(jLabelSelectOutputFolder, org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.jLabelSelectOutputFolder.text")); // NOI18N - jLabelSelectOutputFolder.setVerticalAlignment(javax.swing.SwingConstants.BOTTOM); - javax.swing.GroupLayout nodePanelLayout = new javax.swing.GroupLayout(nodePanel); nodePanel.setLayout(nodePanelLayout); nodePanelLayout.setHorizontalGroup( @@ -302,44 +137,30 @@ public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { .addGroup(nodePanelLayout.createSequentialGroup() .addContainerGap() .addGroup(nodePanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(nodePanelLayout.createSequentialGroup() - .addComponent(outputPathTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 630, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(browseOutputFolderButton)) .addComponent(bnEditIngestSettings, javax.swing.GroupLayout.PREFERRED_SIZE, 155, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(nodePanelLayout.createSequentialGroup() - .addComponent(jLabelSelectOutputFolder) - .addGap(18, 18, 18) - .addComponent(jLabelInvalidResultsFolder, javax.swing.GroupLayout.PREFERRED_SIZE, 544, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addContainerGap(355, Short.MAX_VALUE)) + .addComponent(jLabelDescription, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(bnEditReportSettings, javax.swing.GroupLayout.PREFERRED_SIZE, 155, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); nodePanelLayout.setVerticalGroup( nodePanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(nodePanelLayout.createSequentialGroup() - .addGap(40, 40, 40) - .addGroup(nodePanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(jLabelSelectOutputFolder, javax.swing.GroupLayout.PREFERRED_SIZE, 21, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(jLabelInvalidResultsFolder)) - .addGap(1, 1, 1) - .addGroup(nodePanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(browseOutputFolderButton) - .addComponent(outputPathTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGap(25, 25, 25) + .addGap(27, 27, 27) + .addComponent(jLabelDescription, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(bnEditIngestSettings) - .addContainerGap(389, Short.MAX_VALUE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(bnEditReportSettings) + .addContainerGap(381, Short.MAX_VALUE)) ); - browseOutputFolderButton.getAccessibleContext().setAccessibleName(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.browseOutputFolderButton.text")); // NOI18N - jLabelInvalidResultsFolder.getAccessibleContext().setAccessibleName(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.jLabelInvalidResultsFolder.text")); // NOI18N - jLabelSelectOutputFolder.getAccessibleContext().setAccessibleName(org.openide.util.NbBundle.getMessage(CommandLineIngestSettingsPanel.class, "CommandLineIngestSettingsPanel.jLabelSelectOutputFolder.text")); // NOI18N - nodeScrollPane.setViewportView(nodePanel); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(nodeScrollPane, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, 864, Short.MAX_VALUE) + .addComponent(nodeScrollPane, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -347,54 +168,21 @@ public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { ); }// //GEN-END:initComponents - private void browseOutputFolderButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_browseOutputFolderButtonActionPerformed - String oldText = outputPathTextField.getText().trim(); - // set the current directory of the FileChooser if the oldText is valid - File currentDir = new File(oldText); - if (currentDir.exists()) { - fc.setCurrentDirectory(currentDir); - } - - fc.setDialogTitle("Select case output folder:"); - fc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); - - int retval = fc.showOpenDialog(this); - if (retval == JFileChooser.APPROVE_OPTION) { - String path = fc.getSelectedFile().getPath(); - outputPathTextField.setText(path); - valid(); - controller.changed(); - } - }//GEN-LAST:event_browseOutputFolderButtonActionPerformed + private void bnEditReportSettingsActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_bnEditReportSettingsActionPerformed + this.getParent().setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + doReportWizard(getReportingConfigName(), DISPLAY_CASE_SPECIFIC_DATA, RUN_REPORTS); + this.getParent().setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + }//GEN-LAST:event_bnEditReportSettingsActionPerformed private void bnEditIngestSettingsActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_bnEditIngestSettingsActionPerformed displayIngestJobSettingsPanel(); }//GEN-LAST:event_bnEditIngestSettingsActionPerformed - boolean permissionsAppropriate(String path) { - return FileUtil.hasReadWriteAccess(Paths.get(path)); - } - - private void resetUI() { - load(true); - controller.changed(); - } - - void setEnabledState(boolean enabled) { - bnEditIngestSettings.setEnabled(enabled); - browseOutputFolderButton.setEnabled(enabled); - jLabelInvalidResultsFolder.setEnabled(enabled); - jLabelSelectOutputFolder.setEnabled(enabled); - outputPathTextField.setEnabled(enabled); - } - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton bnEditIngestSettings; - private javax.swing.JButton browseOutputFolderButton; - private javax.swing.JLabel jLabelInvalidResultsFolder; - private javax.swing.JLabel jLabelSelectOutputFolder; + private javax.swing.JButton bnEditReportSettings; + private javax.swing.JLabel jLabelDescription; private javax.swing.JPanel nodePanel; private javax.swing.JScrollPane nodeScrollPane; - private javax.swing.JTextField outputPathTextField; // End of variables declaration//GEN-END:variables } diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanelController.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanelController.java index 7856cdf6e0..a6b019160a 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanelController.java @@ -43,13 +43,11 @@ public final class CommandLineIngestSettingsPanelController extends OptionsPanel @Override public void update() { - getPanel().load(false); changed = false; } @Override public void applyChanges() { - getPanel().store(); changed = false; } @@ -59,7 +57,7 @@ public final class CommandLineIngestSettingsPanelController extends OptionsPanel @Override public boolean isValid() { - return getPanel().valid(); + return true; } @Override diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java index 057ab7b840..387d92293e 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java @@ -19,7 +19,9 @@ package org.sleuthkit.autopsy.commandlineingest; import java.io.File; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; @@ -37,19 +39,36 @@ import org.openide.util.lookup.ServiceProvider; public class CommandLineOptionProcessor extends OptionProcessor { private static final Logger logger = Logger.getLogger(CommandLineOptionProcessor.class.getName()); - private final Option pathToDataSourceOption = Option.optionalArgument('l', "inputPath"); - private final Option caseNameOption = Option.optionalArgument('2', "caseName"); - private final Option runFromCommandLineOption = Option.optionalArgument('3', "runFromCommandLine"); - private String pathToDataSource; - private String baseCaseName; + private final Option caseNameOption = Option.requiredArgument('n', "caseName"); + private final Option caseBaseDirOption = Option.requiredArgument('o', "caseBaseDir"); + private final Option createCaseCommandOption = Option.withoutArgument('c', "createCase"); + private final Option dataSourcePathOption = Option.requiredArgument('s', "dataSourcePath"); + private final Option dataSourceObjectIdOption = Option.requiredArgument('i', "dataSourceObjectId"); + private final Option addDataSourceCommandOption = Option.withoutArgument('a', "addDataSource"); + private final Option caseDirOption = Option.requiredArgument('d', "caseDir"); + private final Option runIngestCommandOption = Option.withoutArgument('r', "runIngest"); + private final Option ingestProfileOption = Option.requiredArgument('p', "ingestProfile"); + private final Option listAllDataSourcesCommandOption = Option.withoutArgument('l', "listAllDataSources"); + private final Option generateReportsOption = Option.withoutArgument('g', "generateReports"); + private boolean runFromCommandLine = false; + private final List commands = new ArrayList<>(); + @Override protected Set
- + - + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - @@ -45,13 +73,16 @@ - + + + + @@ -66,6 +97,9 @@ + + + diff --git a/Core/src/org/sleuthkit/autopsy/communications/CVTTopComponent.java b/Core/src/org/sleuthkit/autopsy/communications/CVTTopComponent.java index 2a5dd8c0d1..a11fde5622 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/CVTTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/communications/CVTTopComponent.java @@ -19,14 +19,16 @@ package org.sleuthkit.autopsy.communications; import com.google.common.eventbus.Subscribe; +import java.awt.BorderLayout; import java.awt.Component; import java.awt.Font; -import java.awt.GridBagConstraints; -import java.awt.GridBagLayout; -import java.awt.Insets; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import javax.swing.ImageIcon; +import javax.swing.JPanel; import javax.swing.JSplitPane; import javax.swing.JTabbedPane; import org.openide.util.Lookup; @@ -35,7 +37,10 @@ import org.openide.windows.Mode; import org.openide.windows.RetainLocation; import org.openide.windows.TopComponent; import org.openide.windows.WindowManager; +import org.sleuthkit.autopsy.communications.relationships.RelationshipBrowser; +import org.sleuthkit.autopsy.communications.relationships.SelectionInfo; import org.sleuthkit.autopsy.coreutils.ThreadConfined; +import org.sleuthkit.datamodel.CommunicationsFilter; /** * Top component which displays the Communications Visualization Tool. @@ -48,10 +53,17 @@ import org.sleuthkit.autopsy.coreutils.ThreadConfined; public final class CVTTopComponent extends TopComponent { private static final long serialVersionUID = 1L; + private boolean filtersVisible = true; + private final RelationshipBrowser relationshipBrowser = new RelationshipBrowser(); + private CommunicationsFilter currentFilter; @ThreadConfined(type = ThreadConfined.ThreadType.AWT) public CVTTopComponent() { initComponents(); + + splitPane.setRightComponent(relationshipBrowser); + splitPane.setDividerLocation(0.25); + setName(Bundle.CVTTopComponent_name()); /* @@ -68,6 +80,8 @@ public final class CVTTopComponent extends TopComponent { Lookup lookup = ((Lookup.Provider)selectedComponent).getLookup(); proxyLookup.setNewLookups(lookup); } + + relationshipBrowser.setSelectionInfo(new SelectionInfo(new HashSet<>(), new HashSet<>(), currentFilter)); }); @@ -80,14 +94,19 @@ public final class CVTTopComponent extends TopComponent { CVTEvents.getCVTEventBus().register(accountsBrowser); CVTEvents.getCVTEventBus().register(filtersPane); - mainSplitPane.setResizeWeight(0.5); - mainSplitPane.setDividerLocation(0.25); + filterTabbedPane.setIconAt(0, new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/arrow-left.png"))); + filterTabbedPane.setTitleAt(0, ""); } @Subscribe void pinAccount(CVTEvents.PinAccountsEvent pinEvent) { browseVisualizeTabPane.setSelectedIndex(1); } + + @Subscribe + void handle(final CVTEvents.FilterChangeEvent filterChangeEvent) { + currentFilter = filterChangeEvent.getNewFilter(); + } /** * This method is called from within the constructor to initialize the form. @@ -96,38 +115,68 @@ public final class CVTTopComponent extends TopComponent { */ // //GEN-BEGIN:initComponents private void initComponents() { - GridBagConstraints gridBagConstraints; - mainSplitPane = new JSplitPane(); + filterTabbedPane = new JTabbedPane(); + filterTabPanel = new JPanel(); filtersPane = new FiltersPanel(); + splitPane = new JSplitPane(); browseVisualizeTabPane = new JTabbedPane(); - accountsBrowser = new AccountsBrowser(); - vizPanel = new VisualizationPanel(); + accountsBrowser = new AccountsBrowser(relationshipBrowser); + vizPanel = new VisualizationPanel(relationshipBrowser); - setLayout(new GridBagLayout()); + setLayout(new BorderLayout()); - mainSplitPane.setLeftComponent(filtersPane); + filterTabbedPane.addMouseListener(new MouseAdapter() { + public void mouseClicked(MouseEvent evt) { + filterTabbedPaneMouseClicked(evt); + } + }); + + filterTabPanel.setLayout(new BorderLayout()); + filterTabPanel.add(filtersPane, BorderLayout.CENTER); + + filterTabbedPane.addTab(NbBundle.getMessage(CVTTopComponent.class, "CVTTopComponent.filterTabPanel.TabConstraints.tabTitle"), filterTabPanel); // NOI18N + + add(filterTabbedPane, BorderLayout.WEST); + + splitPane.setDividerLocation(1); + splitPane.setResizeWeight(0.25); browseVisualizeTabPane.setFont(new Font("Tahoma", 0, 18)); // NOI18N browseVisualizeTabPane.addTab(NbBundle.getMessage(CVTTopComponent.class, "CVTTopComponent.accountsBrowser.TabConstraints.tabTitle_1"), new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/table.png")), accountsBrowser); // NOI18N browseVisualizeTabPane.addTab(NbBundle.getMessage(CVTTopComponent.class, "CVTTopComponent.vizPanel.TabConstraints.tabTitle_1"), new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/emblem-web.png")), vizPanel); // NOI18N - mainSplitPane.setRightComponent(browseVisualizeTabPane); + splitPane.setLeftComponent(browseVisualizeTabPane); browseVisualizeTabPane.getAccessibleContext().setAccessibleName(NbBundle.getMessage(CVTTopComponent.class, "CVTTopComponent.browseVisualizeTabPane.AccessibleContext.accessibleName")); // NOI18N - gridBagConstraints = new GridBagConstraints(); - gridBagConstraints.fill = GridBagConstraints.BOTH; - gridBagConstraints.weightx = 1.0; - gridBagConstraints.weighty = 1.0; - add(mainSplitPane, gridBagConstraints); + add(splitPane, BorderLayout.CENTER); }// //GEN-END:initComponents + private void filterTabbedPaneMouseClicked(MouseEvent evt) {//GEN-FIRST:event_filterTabPaneMouseClicked + int index = filterTabbedPane.indexAtLocation(evt.getX(), evt.getY()); + if(index != -1) { + if(filtersVisible) { + filterTabbedPane.setIconAt(0, new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/arrow-right.png"))); + filterTabPanel.removeAll(); + filterTabPanel.revalidate(); + filtersVisible = false; + } else { + filterTabbedPane.setIconAt(0, new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/arrow-left.png"))); + filterTabPanel.add(filtersPane, BorderLayout.CENTER); + filterTabPanel.revalidate(); + filtersVisible = true; + } + } + }//GEN-LAST:event_filterTabPaneMouseClicked + // Variables declaration - do not modify//GEN-BEGIN:variables private AccountsBrowser accountsBrowser; private JTabbedPane browseVisualizeTabPane; + private JTabbedPane filterTabbedPane; + private JPanel filterTabPanel; private FiltersPanel filtersPane; - private JSplitPane mainSplitPane; + private JSplitPane splitPane; private VisualizationPanel vizPanel; // End of variables declaration//GEN-END:variables diff --git a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.form b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.form index 33fe6d5228..c1b520127e 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.form +++ b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.form @@ -18,6 +18,7 @@ + @@ -37,7 +38,7 @@ - + @@ -128,7 +129,7 @@ - + @@ -222,7 +223,7 @@ - + @@ -325,7 +326,7 @@ - + @@ -422,7 +423,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java index a804735c32..61485832b0 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java +++ b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java @@ -129,9 +129,7 @@ final public class FiltersPanel extends JPanel { public FiltersPanel() { initComponents(); - CheckBoxIconPanel panel = createAccoutTypeCheckBoxPanel(Account.Type.DEVICE, true); - accountTypeMap.put(Account.Type.DEVICE, panel.getCheckBox()); - accountTypeListPane.add(panel); + initalizeDeviceAccountType(); deviceRequiredLabel.setVisible(false); accountTypeRequiredLabel.setVisible(false); @@ -257,9 +255,11 @@ final public class FiltersPanel extends JPanel { //clear the device filter widget when the case changes. devicesMap.clear(); devicesListPane.removeAll(); - - accountTypeMap.clear(); - accountTypeListPane.removeAll(); + + accountTypeMap.clear(); + accountTypeListPane.removeAll(); + + initalizeDeviceAccountType(); }); } @@ -269,6 +269,12 @@ final public class FiltersPanel extends JPanel { IngestManager.getInstance().removeIngestModuleEventListener(ingestListener); IngestManager.getInstance().removeIngestJobEventListener(ingestJobListener); } + + private void initalizeDeviceAccountType() { + CheckBoxIconPanel panel = createAccoutTypeCheckBoxPanel(Account.Type.DEVICE, true); + accountTypeMap.put(Account.Type.DEVICE, panel.getCheckBox()); + accountTypeListPane.add(panel); + } /** * Populate the Account Types filter widgets @@ -471,6 +477,7 @@ final public class FiltersPanel extends JPanel { setLayout(new java.awt.GridBagLayout()); + scrollPane.setAutoscrolls(true); scrollPane.setBorder(null); mainPanel.setLayout(new java.awt.GridBagLayout()); @@ -535,7 +542,7 @@ final public class FiltersPanel extends JPanel { gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; gridBagConstraints.weighty = 1.0; - gridBagConstraints.insets = new java.awt.Insets(15, 0, 15, 0); + gridBagConstraints.insets = new java.awt.Insets(15, 0, 15, 25); mainPanel.add(limitPane, gridBagConstraints); startDatePicker.setEnabled(false); @@ -602,7 +609,7 @@ final public class FiltersPanel extends JPanel { gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; - gridBagConstraints.insets = new java.awt.Insets(15, 0, 0, 0); + gridBagConstraints.insets = new java.awt.Insets(15, 0, 0, 25); mainPanel.add(dateRangePane, gridBagConstraints); devicesPane.setLayout(new java.awt.GridBagLayout()); @@ -680,7 +687,7 @@ final public class FiltersPanel extends JPanel { gridBagConstraints.ipady = 100; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; - gridBagConstraints.insets = new java.awt.Insets(15, 0, 0, 0); + gridBagConstraints.insets = new java.awt.Insets(15, 0, 0, 25); mainPanel.add(devicesPane, gridBagConstraints); accountTypesPane.setLayout(new java.awt.GridBagLayout()); @@ -754,7 +761,7 @@ final public class FiltersPanel extends JPanel { gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; - gridBagConstraints.insets = new java.awt.Insets(15, 0, 0, 0); + gridBagConstraints.insets = new java.awt.Insets(15, 0, 0, 25); mainPanel.add(accountTypesPane, gridBagConstraints); topPane.setLayout(new java.awt.GridBagLayout()); @@ -804,6 +811,7 @@ final public class FiltersPanel extends JPanel { gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; gridBagConstraints.anchor = java.awt.GridBagConstraints.FIRST_LINE_END; gridBagConstraints.weightx = 1.0; + gridBagConstraints.insets = new java.awt.Insets(0, 0, 0, 25); mainPanel.add(topPane, gridBagConstraints); scrollPane.setViewportView(mainPanel); diff --git a/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.form b/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.form index e85901a1ba..18d2d90b0d 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.form +++ b/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.form @@ -11,348 +11,282 @@ - + - - - - - + - + - + - - + + - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.java b/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.java index 8a5404bb1a..263bb200e6 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.java +++ b/Core/src/org/sleuthkit/autopsy/communications/VisualizationPanel.java @@ -43,6 +43,7 @@ import java.awt.BorderLayout; import java.awt.Color; import java.awt.Desktop; import java.awt.Dimension; +import java.awt.FlowLayout; import java.awt.Font; import java.awt.Frame; import java.awt.Graphics; @@ -61,7 +62,6 @@ import java.nio.file.Paths; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -86,7 +86,6 @@ import javax.swing.JMenuItem; import javax.swing.JOptionPane; import javax.swing.JPanel; import javax.swing.JPopupMenu; -import javax.swing.JSplitPane; import javax.swing.JTextArea; import javax.swing.JTextField; import javax.swing.JToolBar; @@ -96,7 +95,6 @@ import javax.swing.SwingWorker; import org.apache.commons.lang3.StringUtils; import org.controlsfx.control.Notifications; import org.jdesktop.layout.GroupLayout; -import org.jdesktop.layout.LayoutStyle; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; @@ -112,6 +110,7 @@ import org.sleuthkit.datamodel.AccountDeviceInstance; import org.sleuthkit.datamodel.CommunicationsFilter; import org.sleuthkit.datamodel.CommunicationsManager; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.autopsy.uicomponents.WrapLayout; /** * A panel that goes in the Visualize tab of the Communications Visualization * Tool. Hosts an JGraphX mxGraphComponent that implements the communications @@ -160,7 +159,8 @@ final public class VisualizationPanel extends JPanel { private final StateManager stateManager; @NbBundle.Messages("VisalizationPanel.paintingError=Problem painting visualization.") - public VisualizationPanel() { + public VisualizationPanel(RelationshipBrowser relationshipBrowser) { + this.relationshipBrowser = relationshipBrowser; initComponents(); //initialize invisible JFXPanel that is used to show JFXNotifications over this window. notificationsJFXPanel.setScene(new Scene(new Pane())); @@ -220,9 +220,6 @@ final public class VisualizationPanel extends JPanel { final GraphMouseListener graphMouseListener = new GraphMouseListener(); graphComponent.getGraphControl().addMouseWheelListener(graphMouseListener); graphComponent.getGraphControl().addMouseListener(graphMouseListener); - - relationshipBrowser = new RelationshipBrowser(); - splitPane.setRightComponent(relationshipBrowser); //feed selection to explorermanager graph.getSelectionModel().addListener(mxEvent.CHANGE, new SelectionListener()); @@ -247,6 +244,8 @@ final public class VisualizationPanel extends JPanel { stateManager = new StateManager(pinnedAccountModel); setStateButtonsEnabled(); + + toolbar.setLayout(new WrapLayout(FlowLayout.LEFT)); } @Subscribe @@ -373,32 +372,28 @@ final public class VisualizationPanel extends JPanel { // //GEN-BEGIN:initComponents private void initComponents() { - splitPane = new JSplitPane(); borderLayoutPanel = new JPanel(); placeHolderPanel = new JPanel(); jTextArea1 = new JTextArea(); - toolbar = new JPanel(); - fastOrganicLayoutButton = new JButton(); - zoomOutButton = new JButton(); - zoomInButton = new JButton(); - zoomActualButton = new JButton(); - fitZoomButton = new JButton(); - zoomLabel = new JLabel(); - zoomPercentLabel = new JLabel(); - clearVizButton = new JButton(); - jSeparator2 = new JToolBar.Separator(); + notificationsJFXPanel = new JFXPanel(); + toolbar = new JToolBar(); backButton = new JButton(); forwardButton = new JButton(); - snapshotButton = new JButton(); jSeparator3 = new JToolBar.Separator(); - jSeparator4 = new JToolBar.Separator(); - notificationsJFXPanel = new JFXPanel(); + clearVizButton = new JButton(); + fastOrganicLayoutButton = new JButton(); + jSeparator2 = new JToolBar.Separator(); + zoomLabel = new JLabel(); + zoomPercentLabel = new JLabel(); + zoomOutButton = new JButton(); + fitZoomButton = new JButton(); + zoomActualButton = new JButton(); + zoomInButton = new JButton(); + jSeparator1 = new JToolBar.Separator(); + snapshotButton = new JButton(); setLayout(new BorderLayout()); - splitPane.setDividerLocation(800); - splitPane.setResizeWeight(0.5); - borderLayoutPanel.setLayout(new BorderLayout()); jTextArea1.setBackground(new Color(240, 240, 240)); @@ -411,9 +406,9 @@ final public class VisualizationPanel extends JPanel { placeHolderPanel.setLayout(placeHolderPanelLayout); placeHolderPanelLayout.setHorizontalGroup(placeHolderPanelLayout.createParallelGroup(GroupLayout.LEADING) .add(placeHolderPanelLayout.createSequentialGroup() - .addContainerGap(250, Short.MAX_VALUE) + .addContainerGap(316, Short.MAX_VALUE) .add(jTextArea1, GroupLayout.PREFERRED_SIZE, 424, GroupLayout.PREFERRED_SIZE) - .addContainerGap(423, Short.MAX_VALUE)) + .addContainerGap(481, Short.MAX_VALUE)) ); placeHolderPanelLayout.setVerticalGroup(placeHolderPanelLayout.createParallelGroup(GroupLayout.LEADING) .add(placeHolderPanelLayout.createSequentialGroup() @@ -423,12 +418,72 @@ final public class VisualizationPanel extends JPanel { ); borderLayoutPanel.add(placeHolderPanel, BorderLayout.CENTER); + borderLayoutPanel.add(notificationsJFXPanel, BorderLayout.PAGE_END); + + add(borderLayoutPanel, BorderLayout.CENTER); + + toolbar.setRollover(true); + + backButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/resultset_previous.png"))); // NOI18N + backButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.backButton.text_1")); // NOI18N + backButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.backButton.toolTipText")); // NOI18N + backButton.setFocusable(false); + backButton.setHorizontalTextPosition(SwingConstants.CENTER); + backButton.setVerticalTextPosition(SwingConstants.BOTTOM); + backButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent evt) { + backButtonActionPerformed(evt); + } + }); + toolbar.add(backButton); + + forwardButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/resultset_next.png"))); // NOI18N + forwardButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.forwardButton.text")); // NOI18N + forwardButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.forwardButton.toolTipText")); // NOI18N + forwardButton.setFocusable(false); + forwardButton.setHorizontalTextPosition(SwingConstants.CENTER); + forwardButton.setVerticalTextPosition(SwingConstants.BOTTOM); + forwardButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent evt) { + forwardButtonActionPerformed(evt); + } + }); + toolbar.add(forwardButton); + toolbar.add(jSeparator3); + + clearVizButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/broom.png"))); // NOI18N + clearVizButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.clearVizButton.text_1")); // NOI18N + clearVizButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.clearVizButton.toolTipText")); // NOI18N + clearVizButton.setActionCommand(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.clearVizButton.actionCommand")); // NOI18N + clearVizButton.setFocusable(false); + clearVizButton.setHorizontalTextPosition(SwingConstants.CENTER); + clearVizButton.setVerticalTextPosition(SwingConstants.BOTTOM); + clearVizButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent evt) { + clearVizButtonActionPerformed(evt); + } + }); + toolbar.add(clearVizButton); fastOrganicLayoutButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/arrow-circle-double-135.png"))); // NOI18N fastOrganicLayoutButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.fastOrganicLayoutButton.text")); // NOI18N fastOrganicLayoutButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.fastOrganicLayoutButton.toolTipText")); // NOI18N fastOrganicLayoutButton.setFocusable(false); + fastOrganicLayoutButton.setHorizontalTextPosition(SwingConstants.CENTER); fastOrganicLayoutButton.setVerticalTextPosition(SwingConstants.BOTTOM); + fastOrganicLayoutButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent evt) { + fastOrganicLayoutButtonActionPerformed(evt); + } + }); + toolbar.add(fastOrganicLayoutButton); + toolbar.add(jSeparator2); + + zoomLabel.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomLabel.text")); // NOI18N + toolbar.add(zoomLabel); + + zoomPercentLabel.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomPercentLabel.text")); // NOI18N + toolbar.add(zoomPercentLabel); zoomOutButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/magnifier-zoom-out-red.png"))); // NOI18N zoomOutButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomOutButton.text")); // NOI18N @@ -441,30 +496,7 @@ final public class VisualizationPanel extends JPanel { zoomOutButtonActionPerformed(evt); } }); - - zoomInButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/magnifier-zoom-in-green.png"))); // NOI18N - zoomInButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomInButton.text")); // NOI18N - zoomInButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomInButton.toolTipText")); // NOI18N - zoomInButton.setFocusable(false); - zoomInButton.setHorizontalTextPosition(SwingConstants.CENTER); - zoomInButton.setVerticalTextPosition(SwingConstants.BOTTOM); - zoomInButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent evt) { - zoomInButtonActionPerformed(evt); - } - }); - - zoomActualButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/magnifier-zoom-actual.png"))); // NOI18N - zoomActualButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomActualButton.text")); // NOI18N - zoomActualButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomActualButton.toolTipText")); // NOI18N - zoomActualButton.setFocusable(false); - zoomActualButton.setHorizontalTextPosition(SwingConstants.CENTER); - zoomActualButton.setVerticalTextPosition(SwingConstants.BOTTOM); - zoomActualButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent evt) { - zoomActualButtonActionPerformed(evt); - } - }); + toolbar.add(zoomOutButton); fitZoomButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/magnifier-zoom-fit.png"))); // NOI18N fitZoomButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.fitZoomButton.text")); // NOI18N @@ -477,116 +509,49 @@ final public class VisualizationPanel extends JPanel { fitZoomButtonActionPerformed(evt); } }); + toolbar.add(fitZoomButton); - zoomLabel.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomLabel.text")); // NOI18N - - zoomPercentLabel.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomPercentLabel.text")); // NOI18N - - clearVizButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/broom.png"))); // NOI18N - clearVizButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.clearVizButton.text_1")); // NOI18N - clearVizButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.clearVizButton.toolTipText")); // NOI18N - clearVizButton.setActionCommand(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.clearVizButton.actionCommand")); // NOI18N - clearVizButton.addActionListener(new ActionListener() { + zoomActualButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/magnifier-zoom-actual.png"))); // NOI18N + zoomActualButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomActualButton.text")); // NOI18N + zoomActualButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomActualButton.toolTipText")); // NOI18N + zoomActualButton.setFocusable(false); + zoomActualButton.setHorizontalTextPosition(SwingConstants.CENTER); + zoomActualButton.setVerticalTextPosition(SwingConstants.BOTTOM); + zoomActualButton.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { - clearVizButtonActionPerformed(evt); + zoomActualButtonActionPerformed(evt); } }); + toolbar.add(zoomActualButton); - jSeparator2.setOrientation(SwingConstants.VERTICAL); - - backButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/resultset_previous.png"))); // NOI18N - backButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.backButton.text_1")); // NOI18N - backButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.backButton.toolTipText")); // NOI18N - backButton.addActionListener(new ActionListener() { + zoomInButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/communications/images/magnifier-zoom-in-green.png"))); // NOI18N + zoomInButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomInButton.text")); // NOI18N + zoomInButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.zoomInButton.toolTipText")); // NOI18N + zoomInButton.setFocusable(false); + zoomInButton.setHorizontalTextPosition(SwingConstants.CENTER); + zoomInButton.setVerticalTextPosition(SwingConstants.BOTTOM); + zoomInButton.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { - backButtonActionPerformed(evt); - } - }); - - forwardButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/resultset_next.png"))); // NOI18N - forwardButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.forwardButton.text")); // NOI18N - forwardButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.forwardButton.toolTipText")); // NOI18N - forwardButton.setHorizontalTextPosition(SwingConstants.LEADING); - forwardButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent evt) { - forwardButtonActionPerformed(evt); + zoomInButtonActionPerformed(evt); } }); + toolbar.add(zoomInButton); + toolbar.add(jSeparator1); snapshotButton.setIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/report/images/image.png"))); // NOI18N snapshotButton.setText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.snapshotButton.text_1")); // NOI18N snapshotButton.setToolTipText(NbBundle.getMessage(VisualizationPanel.class, "VisualizationPanel.snapshotButton.toolTipText")); // NOI18N + snapshotButton.setFocusable(false); + snapshotButton.setHorizontalTextPosition(SwingConstants.CENTER); + snapshotButton.setVerticalTextPosition(SwingConstants.BOTTOM); snapshotButton.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { snapshotButtonActionPerformed(evt); } }); + toolbar.add(snapshotButton); - jSeparator3.setOrientation(SwingConstants.VERTICAL); - - jSeparator4.setOrientation(SwingConstants.VERTICAL); - - GroupLayout toolbarLayout = new GroupLayout(toolbar); - toolbar.setLayout(toolbarLayout); - toolbarLayout.setHorizontalGroup(toolbarLayout.createParallelGroup(GroupLayout.LEADING) - .add(toolbarLayout.createSequentialGroup() - .addContainerGap() - .add(backButton) - .addPreferredGap(LayoutStyle.RELATED) - .add(forwardButton) - .addPreferredGap(LayoutStyle.RELATED) - .add(jSeparator4, GroupLayout.PREFERRED_SIZE, 10, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.RELATED) - .add(fastOrganicLayoutButton) - .addPreferredGap(LayoutStyle.RELATED) - .add(clearVizButton) - .addPreferredGap(LayoutStyle.RELATED) - .add(jSeparator2, GroupLayout.PREFERRED_SIZE, 10, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.RELATED) - .add(zoomLabel) - .addPreferredGap(LayoutStyle.RELATED) - .add(zoomPercentLabel) - .addPreferredGap(LayoutStyle.RELATED) - .add(zoomOutButton, GroupLayout.PREFERRED_SIZE, 32, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.RELATED) - .add(zoomInButton, GroupLayout.PREFERRED_SIZE, 32, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.RELATED) - .add(zoomActualButton, GroupLayout.PREFERRED_SIZE, 33, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.RELATED) - .add(fitZoomButton, GroupLayout.PREFERRED_SIZE, 32, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.RELATED) - .add(jSeparator3, GroupLayout.PREFERRED_SIZE, 10, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.RELATED) - .add(snapshotButton) - .addContainerGap(GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) - ); - toolbarLayout.setVerticalGroup(toolbarLayout.createParallelGroup(GroupLayout.LEADING) - .add(toolbarLayout.createSequentialGroup() - .add(3, 3, 3) - .add(toolbarLayout.createParallelGroup(GroupLayout.CENTER) - .add(fastOrganicLayoutButton) - .add(zoomOutButton) - .add(zoomInButton, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .add(zoomActualButton, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .add(fitZoomButton, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .add(zoomLabel) - .add(zoomPercentLabel) - .add(clearVizButton) - .add(jSeparator2, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .add(backButton) - .add(forwardButton) - .add(snapshotButton) - .add(jSeparator3, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .add(jSeparator4, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) - .add(3, 3, 3)) - ); - - borderLayoutPanel.add(toolbar, BorderLayout.PAGE_START); - borderLayoutPanel.add(notificationsJFXPanel, BorderLayout.PAGE_END); - - splitPane.setLeftComponent(borderLayoutPanel); - - add(splitPane, BorderLayout.CENTER); + add(toolbar, BorderLayout.NORTH); }// //GEN-END:initComponents private void fitZoomButtonActionPerformed(ActionEvent evt) {//GEN-FIRST:event_fitZoomButtonActionPerformed @@ -726,6 +691,10 @@ final public class VisualizationPanel extends JPanel { } }//GEN-LAST:event_snapshotButtonActionPerformed + private void fastOrganicLayoutButtonActionPerformed(ActionEvent evt) {//GEN-FIRST:event_fastOrganicLayoutButtonActionPerformed + // TODO add your handling code here: + }//GEN-LAST:event_fastOrganicLayoutButtonActionPerformed + private void fitGraph() { graphComponent.zoomTo(1, true); mxPoint translate = graph.getView().getTranslate(); @@ -883,15 +852,14 @@ final public class VisualizationPanel extends JPanel { private JButton fastOrganicLayoutButton; private JButton fitZoomButton; private JButton forwardButton; + private JToolBar.Separator jSeparator1; private JToolBar.Separator jSeparator2; private JToolBar.Separator jSeparator3; - private JToolBar.Separator jSeparator4; private JTextArea jTextArea1; private JFXPanel notificationsJFXPanel; private JPanel placeHolderPanel; private JButton snapshotButton; - private JSplitPane splitPane; - private JPanel toolbar; + private JToolBar toolbar; private JButton zoomActualButton; private JButton zoomInButton; private JLabel zoomLabel; diff --git a/Core/src/org/sleuthkit/autopsy/communications/images/arrow-180.png b/Core/src/org/sleuthkit/autopsy/communications/images/arrow-180.png new file mode 100755 index 0000000000..4d2aa3ccb2 Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/communications/images/arrow-180.png differ diff --git a/Core/src/org/sleuthkit/autopsy/communications/images/arrow-left.png b/Core/src/org/sleuthkit/autopsy/communications/images/arrow-left.png new file mode 100755 index 0000000000..e00d990348 Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/communications/images/arrow-left.png differ diff --git a/Core/src/org/sleuthkit/autopsy/communications/images/arrow-right.png b/Core/src/org/sleuthkit/autopsy/communications/images/arrow-right.png new file mode 100755 index 0000000000..cd2c6be8f2 Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/communications/images/arrow-right.png differ diff --git a/Core/src/org/sleuthkit/autopsy/communications/images/arrow.png b/Core/src/org/sleuthkit/autopsy/communications/images/arrow.png new file mode 100755 index 0000000000..12077d3324 Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/communications/images/arrow.png differ diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java new file mode 100755 index 0000000000..5fc7ae8aa7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java @@ -0,0 +1,200 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.communications.relationships; + +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.ImageUtils; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Account; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * + * Class representing the Summary data for a given account. + */ +class AccountSummary { + + private int attachmentCnt; + private int messagesCnt; + private int emailCnt; + private int callLogCnt; + private int contactsCnt; + private int mediaCnt; + private int referenceCnt; + + private final Account selectedAccount; + private final Set artifacts; + + private static final Logger logger = Logger.getLogger(AccountSummary.class.getName()); + + /** + * Summary constructor. + * + * @param selectedAccount Selected account object + * @param artifacts List of relationship source artifacts + */ + AccountSummary(Account selectedAccount, Set artifacts) { + this.selectedAccount = selectedAccount; + this.artifacts = artifacts; + initCounts(); + } + + /** + * Initialize the counts based on the selected account and the given artifacts. + */ + private void initCounts() { + for (BlackboardArtifact artifact : artifacts) { + BlackboardArtifact.ARTIFACT_TYPE fromID = BlackboardArtifact.ARTIFACT_TYPE.fromID(artifact.getArtifactTypeID()); + if (null != fromID) { + switch (fromID) { + case TSK_EMAIL_MSG: + emailCnt++; + break; + case TSK_CALLLOG: + callLogCnt++; + break; + case TSK_MESSAGE: + messagesCnt++; + break; + case TSK_CONTACT: + if (selectedAccount.getAccountType() != Account.Type.DEVICE) { + String typeSpecificID = selectedAccount.getTypeSpecificID(); + + List attributes = null; + + try{ + attributes = artifact.getAttributes(); + } catch(TskCoreException ex) { + logger.log(Level.WARNING, String.format("Unable to getAttributes for artifact: %d", artifact.getArtifactID()), ex); + break; + } + + boolean isReference = false; + + for (BlackboardAttribute attribute: attributes) { + String attributeTypeName = attribute.getAttributeType().getTypeName(); + String attributeValue = attribute.getValueString(); + + if (attributeTypeName.contains("PHONE")) { + attributeValue = RelationshipsNodeUtilities.normalizePhoneNum(attributeValue); + } else if (attributeTypeName.contains("EMAIL")) { + attributeValue = RelationshipsNodeUtilities.normalizeEmailAddress(attributeValue); + } + + if ( typeSpecificID.equals(attributeValue) ) { + isReference = true; + break; + } + } + if (isReference) { + referenceCnt++; + } else { + contactsCnt++; + } + } else { + contactsCnt++; + } + break; + default: + break; + } + } + try { + attachmentCnt += artifact.getChildrenCount(); + for (Content childContent : artifact.getChildren()) { + if (ImageUtils.thumbnailSupported(childContent)) { + mediaCnt++; + } + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Exception thrown " + + "from getChildrenCount artifactID: %d", + artifact.getArtifactID()), ex); //NON-NLS + } + } + } + + /** + * Total number of attachments that this account is referenced. + * + * @return Attachment count + */ + public int getAttachmentCnt() { + return attachmentCnt; + } + + /** + * Total number of messages that this account is referenced. + * + * @return Message count + */ + public int getMessagesCnt() { + return messagesCnt; + } + + /** + * Total number of Emails that this account is referenced. + * + * @return Email count + */ + public int getEmailCnt() { + return emailCnt; + } + + /** + * Total number of call logs that this account is referenced. + * + * @return call log count + */ + public int getCallLogCnt() { + return callLogCnt; + } + + /** + * Total number of contacts in this accounts contact book. + * + * @return contact count + */ + public int getContactsCnt() { + return contactsCnt; + } + + /** + * Total number of thumbnail\media attachments that this account is referenced. + * + * @return Thumbnail count + */ + public int getThumbnailCnt() { + return mediaCnt; + } + + /** + * Total number of contacts that this account is referenced. + * + * @return Contact count + */ + public int getReferenceCnt() { + return referenceCnt; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties index 4d0b858691..01d5316454 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties @@ -1,6 +1,5 @@ ContactDetailsPane.nameLabel.text=Placeholder -SummaryViewer.countsPanel.border.title=Counts -SummaryViewer.contactsLabel.text=Contacts: +SummaryViewer.countsPanel.border.title=Communications OutlineViewPanel.messageLabel.text= SummaryViewer.messagesDataLabel.text=messages SummaryViewer.callLogsDataLabel.text=callLogs @@ -18,6 +17,12 @@ MessageViewer.backButton.AccessibleContext.accessibleDescription= MessageViewer.backButton.text=Threads MessageViewer.showAllButton.text=All Messages SummaryViewer.thumbnailCntLabel.text=Media Attachments: -SummaryViewer.attachmentsLable.text=Total Attachments: SummaryViewer.thumbnailsDataLabel.text=attachments SummaryViewer.attachmentDataLabel.text=count +SummaryViewer.accountLabel.text= +SummaryViewer.contanctsPanel.border.title=Account Contacts +SummaryViewer.accoutDescriptionLabel.text= +SummaryViewer.attachmentsLabel.text=Total Attachments: +SummaryViewer.referencesLabel.text=Communication References: +SummaryViewer.referencesDataLabel.text= +SummaryViewer.contactsLabel.text=Book Entries: diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED index f79bdaa464..3d6fbef0cb 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED @@ -35,20 +35,22 @@ MessageViewer_viewMessage_all=All MessageViewer_viewMessage_calllogs=Call Logs MessageViewer_viewMessage_selected=Selected MessageViewer_viewMessage_unthreaded=Unthreaded -SummaryViewer.countsPanel.border.title=Counts -SummaryViewer.contactsLabel.text=Contacts: +SummaryViewer.countsPanel.border.title=Communications OutlineViewPanel.messageLabel.text= SummaryViewer.messagesDataLabel.text=messages SummaryViewer.callLogsDataLabel.text=callLogs SummaryViewer.contactsDataLabel.text=contacts SummaryViewer.messagesLabel.text=Messages: SummaryViewer.callLogsLabel.text=Call Logs: +SummaryViewer_Account_Description=This account represents a device in the case. +SummaryViewer_Account_Description_MuliSelect=Summary information is not available when multiple accounts are selected. SummaryViewer_CaseRefNameColumn_Title=Case Name -SummaryViewer_CentralRepository_Message= +SummaryViewer_CentralRepository_Message= SummaryViewer_Creation_Date_Title=Creation Date +SummaryViewer_Device_Account_Description=This account was referenced by a device in the case. +SummaryViewer_FileRef_Message= ThreadRootMessagePanel.showAllCheckBox.text=Show All Messages ThreadPane.backButton.text=<--- SummaryViewer.caseReferencesPanel.border.title=Other Occurrences @@ -60,6 +62,12 @@ MessageViewer.backButton.AccessibleContext.accessibleDescription= MessageViewer.backButton.text=Threads MessageViewer.showAllButton.text=All Messages SummaryViewer.thumbnailCntLabel.text=Media Attachments: -SummaryViewer.attachmentsLable.text=Total Attachments: SummaryViewer.thumbnailsDataLabel.text=attachments SummaryViewer.attachmentDataLabel.text=count +SummaryViewer.accountLabel.text= +SummaryViewer.contanctsPanel.border.title=Account Contacts +SummaryViewer.accoutDescriptionLabel.text= +SummaryViewer.attachmentsLabel.text=Total Attachments: +SummaryViewer.referencesLabel.text=Communication References: +SummaryViewer.referencesDataLabel.text= +SummaryViewer.contactsLabel.text=Book Entries: diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/ContactNode.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/ContactNode.java index 33b9d16724..8dbf58acef 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/ContactNode.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/ContactNode.java @@ -18,9 +18,8 @@ */ package org.sleuthkit.autopsy.communications.relationships; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.TimeZone; import java.util.logging.Level; import org.openide.nodes.Sheet; @@ -70,7 +69,7 @@ final class ContactNode extends BlackboardArtifactNode { @Override protected Sheet createSheet() { - Sheet sheet = super.createSheet(); + Sheet sheet = new Sheet(); final BlackboardArtifact artifact = getArtifact(); BlackboardArtifact.ARTIFACT_TYPE fromID = BlackboardArtifact.ARTIFACT_TYPE.fromID(artifact.getArtifactTypeID()); @@ -89,37 +88,33 @@ final class ContactNode extends BlackboardArtifactNode { // are used so that all attributed of that type are found, including // ones that are not predefined as part of BlackboardAttributes try { - HashMap phoneNumMap = new HashMap<>(); - HashMap emailMap = new HashMap<>(); - HashMap nameMap = new HashMap<>(); - HashMap otherMap = new HashMap<>(); + List phoneNumList = new ArrayList<>(); + List emailList = new ArrayList<>(); + List nameList = new ArrayList<>(); + List otherList = new ArrayList<>(); for (BlackboardAttribute bba : artifact.getAttributes()) { if (bba.getAttributeType().getTypeName().startsWith("TSK_PHONE")) { - phoneNumMap.put(bba.getDisplayString(), bba); + phoneNumList.add(bba); } else if (bba.getAttributeType().getTypeName().startsWith("TSK_EMAIL")) { - emailMap.put(bba.getDisplayString(), bba); + emailList.add(bba); } else if (bba.getAttributeType().getTypeName().startsWith("TSK_NAME")) { - nameMap.put(bba.getDisplayString(), bba); + nameList.add(bba); } else { - otherMap.put(bba.getDisplayString(), bba); + otherList.add(bba); } } addPropertiesToSheet(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getLabel(), - sheetSet, nameMap); + sheetSet, nameList); addPropertiesToSheet(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getLabel(), - sheetSet, phoneNumMap); + sheetSet, phoneNumList); addPropertiesToSheet(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getLabel(), - sheetSet, emailMap); + sheetSet, emailList); - for (BlackboardAttribute bba : otherMap.values()) { + for (BlackboardAttribute bba : otherList) { sheetSet.put(new NodeProperty<>(bba.getAttributeType().getTypeName(), bba.getAttributeType().getDisplayName(), "", bba.getDisplayString())); } - // Don't need these values to appear in the Contact property sheet. - sheetSet.remove("S"); - sheetSet.remove("C"); - List children = artifact.getChildren(); if(children != null) { int count = 0; @@ -142,9 +137,9 @@ final class ContactNode extends BlackboardArtifactNode { return sheet; } - private void addPropertiesToSheet(String propertyID, Sheet.Set sheetSet, Map attributeMap) { + private void addPropertiesToSheet(String propertyID, Sheet.Set sheetSet, List attributeList) { int count = 0; - for (BlackboardAttribute bba : attributeMap.values()) { + for (BlackboardAttribute bba : attributeList) { if (count++ > 0) { sheetSet.put(new NodeProperty<>(propertyID + "_" + count, bba.getAttributeType().getDisplayName(), "", bba.getDisplayString())); } else { diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java index db2f6f6ebd..252b4ee1ff 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java @@ -68,4 +68,43 @@ final class RelationshipsNodeUtilities { return ""; } } + + /** + * Normalize the phone number by removing all non numeric characters, except + * for leading +. + * + * This function copied from CommunicationManager. + * + * @param phoneNum The phone number to normalize + * + * @return The normalized phone number. + */ + static String normalizePhoneNum(String phoneNum) { + String normailzedPhoneNum = phoneNum.replaceAll("\\D", ""); + + if (phoneNum.startsWith("+")) { + normailzedPhoneNum = "+" + normailzedPhoneNum; + } + + if (normailzedPhoneNum.isEmpty()) { + normailzedPhoneNum = phoneNum; + } + + return normailzedPhoneNum; + } + + /** + * Normalize the given email address by converting it to lowercase. + * + * This function copied from CommunicationManager. + * + * @param emailAddress The email address tot normalize + * + * @return The normalized email address. + */ + static String normalizeEmailAddress(String emailAddress) { + String normailzedEmailAddr = emailAddress.toLowerCase(); + + return normailzedEmailAddr; + } } diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java index f114ab759c..f9d1d119b7 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java @@ -24,7 +24,6 @@ import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.AccountDeviceInstance; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -47,7 +46,6 @@ public final class SelectionInfo { private final Set accounts; private Set accountArtifacts = null; - private SelectionSummary summary = null; /** * Wraps the details of the currently selected accounts. @@ -150,88 +148,6 @@ public final class SelectionInfo { return accountArtifacts; } - public SelectionSummary getSummary() { - if (summary == null) { - summary = new SelectionSummary(); - } - - return summary; - } - - final class SelectionSummary { - - int attachmentCnt; - int messagesCnt; - int emailCnt; - int callLogCnt; - int contactsCnt; - int mediaCnt; - - SelectionSummary() { - getCounts(); - } - - private void getCounts() { - for (BlackboardArtifact artifact : getArtifacts()) { - BlackboardArtifact.ARTIFACT_TYPE fromID = BlackboardArtifact.ARTIFACT_TYPE.fromID(artifact.getArtifactTypeID()); - if (null != fromID) { - switch (fromID) { - case TSK_EMAIL_MSG: - emailCnt++; - break; - case TSK_CALLLOG: - callLogCnt++; - break; - case TSK_MESSAGE: - messagesCnt++; - break; - case TSK_CONTACT: - contactsCnt++; - break; - default: - break; - } - } - try { - attachmentCnt += artifact.getChildrenCount(); - for (Content childContent : artifact.getChildren()) { - if (ImageUtils.thumbnailSupported(childContent)) { - mediaCnt++; - } - } - } catch (TskCoreException ex) { - logger.log(Level.WARNING, String.format("Exception thrown " - + "from getChildrenCount artifactID: %d", - artifact.getArtifactID()), ex); //NON-NLS - } - } - } - - public int getAttachmentCnt() { - return attachmentCnt; - } - - public int getMessagesCnt() { - return messagesCnt; - } - - public int getEmailCnt() { - return emailCnt; - } - - public int getCallLogCnt() { - return callLogCnt; - } - - public int getContactsCnt() { - return contactsCnt; - } - - public int getThumbnailCnt() { - return mediaCnt; - } - } - /** * Utility class to represent an edge from the graph visualization. */ diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form index 73ef1a68ba..1887560589 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form @@ -1,6 +1,154 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -11,156 +159,11 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -173,7 +176,7 @@ - + @@ -189,9 +192,83 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java index 539c1eb2d3..043b229859 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java @@ -28,7 +28,6 @@ import org.openide.nodes.Children; import org.openide.util.Lookup; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; -import org.sleuthkit.autopsy.communications.relationships.SelectionInfo.SelectionSummary; import org.sleuthkit.datamodel.Account; /** @@ -45,9 +44,13 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi "SummaryViewer_TabTitle=Summary", "SummaryViewer_FileRefNameColumn_Title=Path", "SummaryViewer_CaseRefNameColumn_Title=Case Name", - "SummaryViewer_CentralRepository_Message=", + "SummaryViewer_CentralRepository_Message=", "SummaryViewer_Creation_Date_Title=Creation Date", - "SummeryViewer_FileRef_Message=", + "SummaryViewer_Device_Account_Description=This account was referenced by a device in the case.", + "SummaryViewer_Account_Description=This account represents a device in the case.", + "SummaryViewer_Account_Description_MuliSelect=Summary information is not available when multiple accounts are selected." + }) /** * Creates new form SummaryViewer @@ -72,7 +75,7 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi clearControls(); caseReferencesPanel.hideOutlineView(Bundle.SummaryViewer_CentralRepository_Message()); - fileReferencesPanel.hideOutlineView(Bundle.SummeryViewer_FileRef_Message()); + fileReferencesPanel.hideOutlineView(Bundle.SummaryViewer_FileRef_Message()); } @Override @@ -99,16 +102,31 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi if (info.getAccounts().size() != 1) { setEnabled(false); clearControls(); + + accoutDescriptionLabel.setText(Bundle.SummaryViewer_Account_Description_MuliSelect()); - fileReferencesPanel.hideOutlineView(Bundle.SummeryViewer_FileRef_Message()); + fileReferencesPanel.hideOutlineView(Bundle.SummaryViewer_FileRef_Message()); } else { - SelectionSummary summaryDetails = info.getSummary(); + Account[] accountArray = info.getAccounts().toArray(new Account[1]); + Account account = accountArray[0]; + + accountLabel.setText(account.getTypeSpecificID()); + + if (account.getAccountType().equals(Account.Type.DEVICE)) { + accoutDescriptionLabel.setText(Bundle.SummaryViewer_Account_Description()); + } else { + accoutDescriptionLabel.setText(Bundle.SummaryViewer_Device_Account_Description()); + } + + AccountSummary summaryDetails = new AccountSummary(account, info.getArtifacts()); thumbnailsDataLabel.setText(Integer.toString(summaryDetails.getThumbnailCnt())); callLogsDataLabel.setText(Integer.toString(summaryDetails.getCallLogCnt())); contactsDataLabel.setText(Integer.toString(summaryDetails.getContactsCnt())); messagesDataLabel.setText(Integer.toString(summaryDetails.getMessagesCnt() + summaryDetails.getEmailCnt())); attachmentDataLabel.setText(Integer.toString(summaryDetails.getAttachmentCnt())); + referencesDataLabel.setText(Integer.toString(summaryDetails.getReferenceCnt())); + contactsDataLabel.setText(Integer.toString(summaryDetails.getContactsCnt())); fileReferencesPanel.showOutlineView(); @@ -139,6 +157,8 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi caseReferencesPanel.setEnabled(enabled); fileReferencesPanel.setEnabled(enabled); countsPanel.setEnabled(enabled); + attachmentsLabel.setEnabled(enabled); + referencesLabel.setEnabled(enabled); } /** @@ -150,7 +170,10 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi contactsDataLabel.setText(""); messagesDataLabel.setText(""); attachmentDataLabel.setText(""); - + accountLabel.setText(""); + accoutDescriptionLabel.setText(""); + referencesDataLabel.setText(""); + fileReferencesPanel.setNode(new AbstractNode(Children.LEAF)); caseReferencesPanel.setNode(new AbstractNode(Children.LEAF)); } @@ -185,132 +208,204 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi private void initComponents() { java.awt.GridBagConstraints gridBagConstraints; + summaryPanel = new javax.swing.JPanel(); + accountLabel = new javax.swing.JLabel(); + accoutDescriptionLabel = new javax.swing.JLabel(); countsPanel = new javax.swing.JPanel(); - contactsLabel = new javax.swing.JLabel(); messagesLabel = new javax.swing.JLabel(); callLogsLabel = new javax.swing.JLabel(); thumbnailCntLabel = new javax.swing.JLabel(); thumbnailsDataLabel = new javax.swing.JLabel(); messagesDataLabel = new javax.swing.JLabel(); callLogsDataLabel = new javax.swing.JLabel(); - contactsDataLabel = new javax.swing.JLabel(); - attachmentsLable = new javax.swing.JLabel(); + attachmentsLabel = new javax.swing.JLabel(); attachmentDataLabel = new javax.swing.JLabel(); fileReferencesPanel = new org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel(); caseReferencesPanel = new org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel(); + contanctsPanel = new javax.swing.JPanel(); + contactsLabel = new javax.swing.JLabel(); + contactsDataLabel = new javax.swing.JLabel(); + referencesLabel = new javax.swing.JLabel(); + referencesDataLabel = new javax.swing.JLabel(); - setLayout(new java.awt.GridBagLayout()); - - countsPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.countsPanel.border.title"))); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(contactsLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.contactsLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(messagesLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.messagesLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(callLogsLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.callLogsLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(thumbnailCntLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.thumbnailCntLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(thumbnailsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.thumbnailsDataLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(messagesDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.messagesDataLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(callLogsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.callLogsDataLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(contactsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.contactsDataLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(attachmentsLable, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentsLable.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(attachmentDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentDataLabel.text")); // NOI18N - - javax.swing.GroupLayout countsPanelLayout = new javax.swing.GroupLayout(countsPanel); - countsPanel.setLayout(countsPanelLayout); - countsPanelLayout.setHorizontalGroup( - countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(countsPanelLayout.createSequentialGroup() - .addContainerGap() - .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(messagesLabel) - .addComponent(callLogsLabel) - .addComponent(contactsLabel) - .addComponent(thumbnailCntLabel) - .addComponent(attachmentsLable)) - .addGap(18, 18, 18) - .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(attachmentDataLabel) - .addComponent(thumbnailsDataLabel) - .addComponent(contactsDataLabel) - .addComponent(callLogsDataLabel) - .addComponent(messagesDataLabel)) - .addContainerGap(845, Short.MAX_VALUE)) - ); - countsPanelLayout.setVerticalGroup( - countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(countsPanelLayout.createSequentialGroup() - .addGap(7, 7, 7) - .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(messagesLabel) - .addComponent(messagesDataLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(callLogsLabel) - .addComponent(callLogsDataLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(contactsLabel) - .addComponent(contactsDataLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(thumbnailCntLabel) - .addComponent(thumbnailsDataLabel)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(attachmentsLable) - .addComponent(attachmentDataLabel))) - ); + summaryPanel.setLayout(new java.awt.GridBagLayout()); + org.openide.awt.Mnemonics.setLocalizedText(accountLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.accountLabel.text")); // NOI18N gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 0; gridBagConstraints.gridy = 0; - gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(15, 9, 0, 9); + summaryPanel.add(accountLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(accoutDescriptionLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.accoutDescriptionLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 1; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; - add(countsPanel, gridBagConstraints); + gridBagConstraints.insets = new java.awt.Insets(15, 9, 15, 9); + summaryPanel.add(accoutDescriptionLabel, gridBagConstraints); + + countsPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.countsPanel.border.title"))); // NOI18N + countsPanel.setLayout(new java.awt.GridBagLayout()); + + org.openide.awt.Mnemonics.setLocalizedText(messagesLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.messagesLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(9, 15, 9, 15); + countsPanel.add(messagesLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(callLogsLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.callLogsLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 1; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 15, 9, 15); + countsPanel.add(callLogsLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(thumbnailCntLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.thumbnailCntLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 2; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 15, 9, 15); + countsPanel.add(thumbnailCntLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(thumbnailsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.thumbnailsDataLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 2; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 0, 9, 15); + countsPanel.add(thumbnailsDataLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(messagesDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.messagesDataLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.weightx = 1.0; + gridBagConstraints.insets = new java.awt.Insets(9, 0, 9, 15); + countsPanel.add(messagesDataLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(callLogsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.callLogsDataLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 1; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 0, 9, 15); + countsPanel.add(callLogsDataLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(attachmentsLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentsLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 3; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 15, 9, 15); + countsPanel.add(attachmentsLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(attachmentDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentDataLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 3; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 0, 9, 15); + countsPanel.add(attachmentDataLabel, gridBagConstraints); + + summaryPanel.add(countsPanel, new java.awt.GridBagConstraints()); + + setLayout(new java.awt.GridBagLayout()); fileReferencesPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.fileReferencesPanel.border.title"))); // NOI18N gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 0; - gridBagConstraints.gridy = 1; + gridBagConstraints.gridy = 3; gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(9, 0, 0, 0); add(fileReferencesPanel, gridBagConstraints); caseReferencesPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.caseReferencesPanel.border.title"))); // NOI18N gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 0; - gridBagConstraints.gridy = 2; + gridBagConstraints.gridy = 4; gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(9, 0, 0, 0); add(caseReferencesPanel, gridBagConstraints); + + contanctsPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.contanctsPanel.border.title"))); // NOI18N + contanctsPanel.setLayout(new java.awt.GridBagLayout()); + + contactsLabel.setLabelFor(contactsDataLabel); + org.openide.awt.Mnemonics.setLocalizedText(contactsLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.contactsLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(9, 15, 9, 15); + contanctsPanel.add(contactsLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(contactsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.contactsDataLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.weightx = 1.0; + gridBagConstraints.insets = new java.awt.Insets(9, 9, 9, 15); + contanctsPanel.add(contactsDataLabel, gridBagConstraints); + + referencesLabel.setLabelFor(referencesDataLabel); + org.openide.awt.Mnemonics.setLocalizedText(referencesLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.referencesLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 1; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 15, 9, 0); + contanctsPanel.add(referencesLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(referencesDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.referencesDataLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 1; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.insets = new java.awt.Insets(0, 9, 0, 0); + contanctsPanel.add(referencesDataLabel, gridBagConstraints); + + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 1; + gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + add(contanctsPanel, gridBagConstraints); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JLabel accountLabel; + private javax.swing.JLabel accoutDescriptionLabel; private javax.swing.JLabel attachmentDataLabel; - private javax.swing.JLabel attachmentsLable; + private javax.swing.JLabel attachmentsLabel; private javax.swing.JLabel callLogsDataLabel; private javax.swing.JLabel callLogsLabel; private org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel caseReferencesPanel; private javax.swing.JLabel contactsDataLabel; private javax.swing.JLabel contactsLabel; + private javax.swing.JPanel contanctsPanel; private javax.swing.JPanel countsPanel; private org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel fileReferencesPanel; private javax.swing.JLabel messagesDataLabel; private javax.swing.JLabel messagesLabel; + private javax.swing.JLabel referencesDataLabel; + private javax.swing.JLabel referencesLabel; + private javax.swing.JPanel summaryPanel; private javax.swing.JLabel thumbnailCntLabel; private javax.swing.JLabel thumbnailsDataLabel; // End of variables declaration//GEN-END:variables diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/FileViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/FileViewer.java index dddf0f6296..1b0d048f35 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/FileViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/FileViewer.java @@ -117,7 +117,7 @@ public class FileViewer extends javax.swing.JPanel implements DataContentViewer } AbstractFile file = selectedNode.getLookup().lookup(AbstractFile.class); - if (file == null) { + if ((file == null) || (file.isDir())) { return; } @@ -189,7 +189,7 @@ public class FileViewer extends javax.swing.JPanel implements DataContentViewer } AbstractFile aFile = node.getLookup().lookup(AbstractFile.class); - if (aFile == null) { + if ((aFile == null) || (aFile.isDir())) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaPlayerPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaPlayerPanel.java index 5e0c85e807..1b30c49ef4 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaPlayerPanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaPlayerPanel.java @@ -187,7 +187,6 @@ public class MediaPlayerPanel extends JPanel implements MediaFileViewer.MediaVie */ public MediaPlayerPanel() throws GstException, UnsatisfiedLinkError { initComponents(); - initGst(); customizeComponents(); } @@ -251,11 +250,6 @@ public class MediaPlayerPanel extends JPanel implements MediaFileViewer.MediaVie }; } - private void initGst() throws GstException, UnsatisfiedLinkError { - logger.log(Level.INFO, "Attempting initializing of gstreamer for video/audio viewing"); //NON-NLS - Gst.init(); - } - /** * Loads the file by spawning off a background task to handle file copying * and video component initializations. @@ -453,6 +447,12 @@ public class MediaPlayerPanel extends JPanel implements MediaFileViewer.MediaVie if(this.isCancelled()) { return; } + + // Initialize Gstreamer. It is safe to call this for every file. + // It was moved here from the constructor because having it happen + // earlier resulted in conflicts on Linux. + Gst.init(); + //Video is ready for playback. Create new components gstPlayBin = new PlayBin("VideoPlayer", tempFile.toURI()); //Configure event handling diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java index 0e1b9c9fb1..e5793e9c3e 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java @@ -199,7 +199,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan imageTaggingOptions.setPopupSize(300, 150); //Disable image tagging for non-windows users or upon failure to load OpenCV. - if (!PlatformUtil.isWindowsOS() || !OpenCvLoader.hasOpenCvLoaded()) { + if (!PlatformUtil.isWindowsOS() || !OpenCvLoader.openCvIsLoaded()) { tagsMenu.setEnabled(false); imageTaggingOptions.setEnabled(false); } diff --git a/Core/src/org/sleuthkit/autopsy/core/layer.xml b/Core/src/org/sleuthkit/autopsy/core/layer.xml index 41788864d6..6a9673a6d3 100644 --- a/Core/src/org/sleuthkit/autopsy/core/layer.xml +++ b/Core/src/org/sleuthkit/autopsy/core/layer.xml @@ -150,16 +150,16 @@ + + + + - + - - - - @@ -220,8 +220,8 @@ - diff --git a/KeywordSearch/nbproject/project.properties b/KeywordSearch/nbproject/project.properties index 1041bdd524..72a5c81ab7 100644 --- a/KeywordSearch/nbproject/project.properties +++ b/KeywordSearch/nbproject/project.properties @@ -29,6 +29,7 @@ file.reference.jericho-html-3.3.jar=release/modules/ext/jericho-html-3.3.jar file.reference.joda-time-2.2.jar=release/modules/ext/joda-time-2.2.jar file.reference.json-simple-1.1.1.jar=release/modules/ext/json-simple-1.1.1.jar file.reference.juniversalchardet-1.0.3.jar=release/modules/ext/juniversalchardet-1.0.3.jar +file.reference.language-detector-0.6.jar=release\\modules\\ext\\language-detector-0.6.jar file.reference.libsvm-3.1.jar=release/modules/ext/libsvm-3.1.jar file.reference.log4j-1.2.17.jar=release/modules/ext/log4j-1.2.17.jar file.reference.lucene-core-4.0.0.jar=release/modules/ext/lucene-core-4.0.0.jar diff --git a/KeywordSearch/nbproject/project.xml b/KeywordSearch/nbproject/project.xml index 3f40ab3ace..78b0b2626a 100644 --- a/KeywordSearch/nbproject/project.xml +++ b/KeywordSearch/nbproject/project.xml @@ -119,7 +119,7 @@ 10 - 10.16 + 10.17 @@ -128,7 +128,7 @@ 3 - 1.2 + 1.3 @@ -230,10 +230,6 @@ org.codehaus.stax2.validation org.noggit org.sleuthkit.autopsy.keywordsearch - org.slf4j - org.slf4j.event - org.slf4j.helpers - org.slf4j.spi ext/commons-digester-1.8.1.jar @@ -283,6 +279,10 @@ ext/guava-17.0.jar release/modules/ext/guava-17.0.jar + + ext/language-detector-0.6.jar + release\modules\ext\language-detector-0.6.jar + ext/joda-time-2.2.jar release/modules/ext/joda-time-2.2.jar diff --git a/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/lang/stoptags_ja.txt b/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/lang/stoptags_ja.txt new file mode 100755 index 0000000000..71b750845e --- /dev/null +++ b/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/lang/stoptags_ja.txt @@ -0,0 +1,420 @@ +# +# This file defines a Japanese stoptag set for JapanesePartOfSpeechStopFilter. +# +# Any token with a part-of-speech tag that exactly matches those defined in this +# file are removed from the token stream. +# +# Set your own stoptags by uncommenting the lines below. Note that comments are +# not allowed on the same line as a stoptag. See LUCENE-3745 for frequency lists, +# etc. that can be useful for building you own stoptag set. +# +# The entire possible tagset is provided below for convenience. +# +##### +# noun: unclassified nouns +#名詞 +# +# noun-common: Common nouns or nouns where the sub-classification is undefined +#名詞-一般 +# +# noun-proper: Proper nouns where the sub-classification is undefined +#名詞-固有名詞 +# +# noun-proper-misc: miscellaneous proper nouns +#名詞-固有名詞-一般 +# +# noun-proper-person: Personal names where the sub-classification is undefined +#名詞-固有名詞-人名 +# +# noun-proper-person-misc: names that cannot be divided into surname and +# given name; foreign names; names where the surname or given name is unknown. +# e.g. お市の方 +#名詞-固有名詞-人名-一般 +# +# noun-proper-person-surname: Mainly Japanese surnames. +# e.g. 山田 +#名詞-固有名詞-人名-姓 +# +# noun-proper-person-given_name: Mainly Japanese given names. +# e.g. 太郎 +#名詞-固有名詞-人名-名 +# +# noun-proper-organization: Names representing organizations. +# e.g. 通産省, NHK +#名詞-固有名詞-組織 +# +# noun-proper-place: Place names where the sub-classification is undefined +#名詞-固有名詞-地域 +# +# noun-proper-place-misc: Place names excluding countries. +# e.g. アジア, バルセロナ, 京都 +#名詞-固有名詞-地域-一般 +# +# noun-proper-place-country: Country names. +# e.g. 日本, オーストラリア +#名詞-固有名詞-地域-国 +# +# noun-pronoun: Pronouns where the sub-classification is undefined +#名詞-代名詞 +# +# noun-pronoun-misc: miscellaneous pronouns: +# e.g. それ, ここ, あいつ, あなた, あちこち, いくつ, どこか, なに, みなさん, みんな, わたくし, われわれ +#名詞-代名詞-一般 +# +# noun-pronoun-contraction: Spoken language contraction made by combining a +# pronoun and the particle 'wa'. +# e.g. ありゃ, こりゃ, こりゃあ, そりゃ, そりゃあ +#名詞-代名詞-縮約 +# +# noun-adverbial: Temporal nouns such as names of days or months that behave +# like adverbs. Nouns that represent amount or ratios and can be used adverbially, +# e.g. 金曜, 一月, 午後, 少量 +#名詞-副詞可能 +# +# noun-verbal: Nouns that take arguments with case and can appear followed by +# 'suru' and related verbs (する, できる, なさる, くださる) +# e.g. インプット, 愛着, 悪化, 悪戦苦闘, 一安心, 下取り +#名詞-サ変接続 +# +# noun-adjective-base: The base form of adjectives, words that appear before な ("na") +# e.g. 健康, 安易, 駄目, だめ +#名詞-形容動詞語幹 +# +# noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), 数. +# e.g. 0, 1, 2, 何, 数, 幾 +#名詞-数 +# +# noun-affix: noun affixes where the sub-classification is undefined +#名詞-非自立 +# +# noun-affix-misc: Of adnominalizers, the case-marker の ("no"), and words that +# attach to the base form of inflectional words, words that cannot be classified +# into any of the other categories below. This category includes indefinite nouns. +# e.g. あかつき, 暁, かい, 甲斐, 気, きらい, 嫌い, くせ, 癖, こと, 事, ごと, 毎, しだい, 次第, +# 順, せい, 所為, ついで, 序で, つもり, 積もり, 点, どころ, の, はず, 筈, はずみ, 弾み, +# 拍子, ふう, ふり, 振り, ほう, 方, 旨, もの, 物, 者, ゆえ, 故, ゆえん, 所以, わけ, 訳, +# わり, 割り, 割, ん-口語/, もん-口語/ +#名詞-非自立-一般 +# +# noun-affix-adverbial: noun affixes that that can behave as adverbs. +# e.g. あいだ, 間, あげく, 挙げ句, あと, 後, 余り, 以外, 以降, 以後, 以上, 以前, 一方, うえ, +# 上, うち, 内, おり, 折り, かぎり, 限り, きり, っきり, 結果, ころ, 頃, さい, 際, 最中, さなか, +# 最中, じたい, 自体, たび, 度, ため, 為, つど, 都度, とおり, 通り, とき, 時, ところ, 所, +# とたん, 途端, なか, 中, のち, 後, ばあい, 場合, 日, ぶん, 分, ほか, 他, まえ, 前, まま, +# 儘, 侭, みぎり, 矢先 +#名詞-非自立-副詞可能 +# +# noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars +# with the stem よう(だ) ("you(da)"). +# e.g. よう, やう, 様 (よう) +#名詞-非自立-助動詞語幹 +# +# noun-affix-adjective-base: noun affixes that can connect to the indeclinable +# connection form な (aux "da"). +# e.g. みたい, ふう +#名詞-非自立-形容動詞語幹 +# +# noun-special: special nouns where the sub-classification is undefined. +#名詞-特殊 +# +# noun-special-aux: The そうだ ("souda") stem form that is used for reporting news, is +# treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base +# form of inflectional words. +# e.g. そう +#名詞-特殊-助動詞語幹 +# +# noun-suffix: noun suffixes where the sub-classification is undefined. +#名詞-接尾 +# +# noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect +# to ガル or タイ and can combine into compound nouns, words that cannot be classified into +# any of the other categories below. In general, this category is more inclusive than +# 接尾語 ("suffix") and is usually the last element in a compound noun. +# e.g. おき, かた, 方, 甲斐 (がい), がかり, ぎみ, 気味, ぐるみ, (~した) さ, 次第, 済 (ず) み, +# よう, (でき)っこ, 感, 観, 性, 学, 類, 面, 用 +#名詞-接尾-一般 +# +# noun-suffix-person: Suffixes that form nouns and attach to person names more often +# than other nouns. +# e.g. 君, 様, 著 +#名詞-接尾-人名 +# +# noun-suffix-place: Suffixes that form nouns and attach to place names more often +# than other nouns. +# e.g. 町, 市, 県 +#名詞-接尾-地域 +# +# noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that +# can appear before スル ("suru"). +# e.g. 化, 視, 分け, 入り, 落ち, 買い +#名詞-接尾-サ変接続 +# +# noun-suffix-aux: The stem form of そうだ (様態) that is used to indicate conditions, +# is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the +# conjunctive form of inflectional words. +# e.g. そう +#名詞-接尾-助動詞語幹 +# +# noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive +# form of inflectional words and appear before the copula だ ("da"). +# e.g. 的, げ, がち +#名詞-接尾-形容動詞語幹 +# +# noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs. +# e.g. 後 (ご), 以後, 以降, 以前, 前後, 中, 末, 上, 時 (じ) +#名詞-接尾-副詞可能 +# +# noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category +# is more inclusive than 助数詞 ("classifier") and includes common nouns that attach +# to numbers. +# e.g. 個, つ, 本, 冊, パーセント, cm, kg, カ月, か国, 区画, 時間, 時半 +#名詞-接尾-助数詞 +# +# noun-suffix-special: Special suffixes that mainly attach to inflecting words. +# e.g. (楽し) さ, (考え) 方 +#名詞-接尾-特殊 +# +# noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words +# together. +# e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) 兼 (主婦) +#名詞-接続詞的 +# +# noun-verbal_aux: Nouns that attach to the conjunctive particle て ("te") and are +# semantically verb-like. +# e.g. ごらん, ご覧, 御覧, 頂戴 +#名詞-動詞非自立的 +# +# noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry, +# dialects, English, etc. Currently, the only entry for 名詞 引用文字列 ("noun quotation") +# is いわく ("iwaku"). +#名詞-引用文字列 +# +# noun-nai_adjective: Words that appear before the auxiliary verb ない ("nai") and +# behave like an adjective. +# e.g. 申し訳, 仕方, とんでも, 違い +#名詞-ナイ形容詞語幹 +# +##### +# prefix: unclassified prefixes +#接頭詞 +# +# prefix-nominal: Prefixes that attach to nouns (including adjective stem forms) +# excluding numerical expressions. +# e.g. お (水), 某 (氏), 同 (社), 故 (~氏), 高 (品質), お (見事), ご (立派) +#接頭詞-名詞接続 +# +# prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb +# in conjunctive form followed by なる/なさる/くださる. +# e.g. お (読みなさい), お (座り) +#接頭詞-動詞接続 +# +# prefix-adjectival: Prefixes that attach to adjectives. +# e.g. お (寒いですねえ), バカ (でかい) +#接頭詞-形容詞接続 +# +# prefix-numerical: Prefixes that attach to numerical expressions. +# e.g. 約, およそ, 毎時 +#接頭詞-数接続 +# +##### +# verb: unclassified verbs +#動詞 +# +# verb-main: +#動詞-自立 +# +# verb-auxiliary: +#動詞-非自立 +# +# verb-suffix: +#動詞-接尾 +# +##### +# adjective: unclassified adjectives +#形容詞 +# +# adjective-main: +#形容詞-自立 +# +# adjective-auxiliary: +#形容詞-非自立 +# +# adjective-suffix: +#形容詞-接尾 +# +##### +# adverb: unclassified adverbs +#副詞 +# +# adverb-misc: Words that can be segmented into one unit and where adnominal +# modification is not possible. +# e.g. あいかわらず, 多分 +#副詞-一般 +# +# adverb-particle_conjunction: Adverbs that can be followed by の, は, に, +# な, する, だ, etc. +# e.g. こんなに, そんなに, あんなに, なにか, なんでも +#副詞-助詞類接続 +# +##### +# adnominal: Words that only have noun-modifying forms. +# e.g. この, その, あの, どの, いわゆる, なんらかの, 何らかの, いろんな, こういう, そういう, ああいう, +# どういう, こんな, そんな, あんな, どんな, 大きな, 小さな, おかしな, ほんの, たいした, +# 「(, も) さる (ことながら)」, 微々たる, 堂々たる, 単なる, いかなる, 我が」「同じ, 亡き +#連体詞 +# +##### +# conjunction: Conjunctions that can occur independently. +# e.g. が, けれども, そして, じゃあ, それどころか +接続詞 +# +##### +# particle: unclassified particles. +助詞 +# +# particle-case: case particles where the subclassification is undefined. +助詞-格助詞 +# +# particle-case-misc: Case particles. +# e.g. から, が, で, と, に, へ, より, を, の, にて +助詞-格助詞-一般 +# +# particle-case-quote: the "to" that appears after nouns, a person’s speech, +# quotation marks, expressions of decisions from a meeting, reasons, judgements, +# conjectures, etc. +# e.g. ( だ) と (述べた.), ( である) と (して執行猶予...) +助詞-格助詞-引用 +# +# particle-case-compound: Compounds of particles and verbs that mainly behave +# like case particles. +# e.g. という, といった, とかいう, として, とともに, と共に, でもって, にあたって, に当たって, に当って, +# にあたり, に当たり, に当り, に当たる, にあたる, において, に於いて,に於て, における, に於ける, +# にかけ, にかけて, にかんし, に関し, にかんして, に関して, にかんする, に関する, に際し, +# に際して, にしたがい, に従い, に従う, にしたがって, に従って, にたいし, に対し, にたいして, +# に対して, にたいする, に対する, について, につき, につけ, につけて, につれ, につれて, にとって, +# にとり, にまつわる, によって, に依って, に因って, により, に依り, に因り, による, に依る, に因る, +# にわたって, にわたる, をもって, を以って, を通じ, を通じて, を通して, をめぐって, をめぐり, をめぐる, +# って-口語/, ちゅう-関西弁「という」/, (何) ていう (人)-口語/, っていう-口語/, といふ, とかいふ +助詞-格助詞-連語 +# +# particle-conjunctive: +# e.g. から, からには, が, けれど, けれども, けど, し, つつ, て, で, と, ところが, どころか, とも, ども, +# ながら, なり, ので, のに, ば, ものの, や ( した), やいなや, (ころん) じゃ(いけない)-口語/, +# (行っ) ちゃ(いけない)-口語/, (言っ) たって (しかたがない)-口語/, (それがなく)ったって (平気)-口語/ +助詞-接続助詞 +# +# particle-dependency: +# e.g. こそ, さえ, しか, すら, は, も, ぞ +助詞-係助詞 +# +# particle-adverbial: +# e.g. がてら, かも, くらい, 位, ぐらい, しも, (学校) じゃ(これが流行っている)-口語/, +# (それ)じゃあ (よくない)-口語/, ずつ, (私) なぞ, など, (私) なり (に), (先生) なんか (大嫌い)-口語/, +# (私) なんぞ, (先生) なんて (大嫌い)-口語/, のみ, だけ, (私) だって-口語/, だに, +# (彼)ったら-口語/, (お茶) でも (いかが), 等 (とう), (今後) とも, ばかり, ばっか-口語/, ばっかり-口語/, +# ほど, 程, まで, 迄, (誰) も (が)([助詞-格助詞] および [助詞-係助詞] の前に位置する「も」) +助詞-副助詞 +# +# particle-interjective: particles with interjective grammatical roles. +# e.g. (松島) や +助詞-間投助詞 +# +# particle-coordinate: +# e.g. と, たり, だの, だり, とか, なり, や, やら +助詞-並立助詞 +# +# particle-final: +# e.g. かい, かしら, さ, ぜ, (だ)っけ-口語/, (とまってる) で-方言/, な, ナ, なあ-口語/, ぞ, ね, ネ, +# ねぇ-口語/, ねえ-口語/, ねん-方言/, の, のう-口語/, や, よ, ヨ, よぉ-口語/, わ, わい-口語/ +助詞-終助詞 +# +# particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is +# adverbial, conjunctive, or sentence final. For example: +# (a) 「A か B か」. Ex:「(国内で運用する) か,(海外で運用する) か (.)」 +# (b) Inside an adverb phrase. Ex:「(幸いという) か (, 死者はいなかった.)」 +# 「(祈りが届いたせい) か (, 試験に合格した.)」 +# (c) 「かのように」. Ex:「(何もなかった) か (のように振る舞った.)」 +# e.g. か +助詞-副助詞/並立助詞/終助詞 +# +# particle-adnominalizer: The "no" that attaches to nouns and modifies +# non-inflectional words. +助詞-連体化 +# +# particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs +# that are giongo, giseigo, or gitaigo. +# e.g. に, と +助詞-副詞化 +# +# particle-special: A particle that does not fit into one of the above classifications. +# This includes particles that are used in Tanka, Haiku, and other poetry. +# e.g. かな, けむ, ( しただろう) に, (あんた) にゃ(わからん), (俺) ん (家) +助詞-特殊 +# +##### +# auxiliary-verb: +助動詞 +# +##### +# interjection: Greetings and other exclamations. +# e.g. おはよう, おはようございます, こんにちは, こんばんは, ありがとう, どうもありがとう, ありがとうございます, +# いただきます, ごちそうさま, さよなら, さようなら, はい, いいえ, ごめん, ごめんなさい +#感動詞 +# +##### +# symbol: unclassified Symbols. +記号 +# +# symbol-misc: A general symbol not in one of the categories below. +# e.g. [○◎@$〒→+] +記号-一般 +# +# symbol-comma: Commas +# e.g. [,、] +記号-読点 +# +# symbol-period: Periods and full stops. +# e.g. [..。] +記号-句点 +# +# symbol-space: Full-width whitespace. +記号-空白 +# +# symbol-open_bracket: +# e.g. [({‘“『【] +記号-括弧開 +# +# symbol-close_bracket: +# e.g. [)}’”』」】] +記号-括弧閉 +# +# symbol-alphabetic: +#記号-アルファベット +# +##### +# other: unclassified other +#その他 +# +# other-interjection: Words that are hard to classify as noun-suffixes or +# sentence-final particles. +# e.g. (だ)ァ +その他-間投 +# +##### +# filler: Aizuchi that occurs during a conversation or sounds inserted as filler. +# e.g. あの, うんと, えと +フィラー +# +##### +# non-verbal: non-verbal sound. +非言語音 +# +##### +# fragment: +#語断片 +# +##### +# unknown: unknown part of speech. +#未知語 +# +##### End of file diff --git a/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/lang/stopwords_ja.txt b/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/lang/stopwords_ja.txt new file mode 100755 index 0000000000..d4321be6b1 --- /dev/null +++ b/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/lang/stopwords_ja.txt @@ -0,0 +1,127 @@ +# +# This file defines a stopword set for Japanese. +# +# This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. +# Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 +# for frequency lists, etc. that can be useful for making your own set (if desired) +# +# Note that there is an overlap between these stopwords and the terms stopped when used +# in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note +# that comments are not allowed on the same line as stopwords. +# +# Also note that stopping is done in a case-insensitive manner. Change your StopFilter +# configuration if you need case-sensitive stopping. Lastly, note that stopping is done +# using the same character width as the entries in this file. Since this StopFilter is +# normally done after a CJKWidthFilter in your chain, you would usually want your romaji +# entries to be in half-width and your kana entries to be in full-width. +# +の +に +は +を +た +が +で +て +と +し +れ +さ +ある +いる +も +する +から +な +こと +として +い +や +れる +など +なっ +ない +この +ため +その +あっ +よう +また +もの +という +あり +まで +られ +なる +へ +か +だ +これ +によって +により +おり +より +による +ず +なり +られる +において +ば +なかっ +なく +しかし +について +せ +だっ +その後 +できる +それ +う +ので +なお +のみ +でき +き +つ +における +および +いう +さらに +でも +ら +たり +その他 +に関する +たち +ます +ん +なら +に対して +特に +せる +及び +これら +とき +では +にて +ほか +ながら +うち +そして +とともに +ただし +かつて +それぞれ +または +お +ほど +ものの +に対する +ほとんど +と共に +といった +です +とも +ところ +ここ +##### End of file diff --git a/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/schema.xml b/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/schema.xml index 05ea8891a5..bbc68fea00 100644 --- a/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/schema.xml +++ b/KeywordSearch/solr/solr/configsets/AutopsyConfig/conf/schema.xml @@ -45,7 +45,7 @@ that avoids logging every request --> - + @@ -243,6 +244,18 @@ + + + + + + + + + + + + + + + + + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index e5c93303b3..5cfd965ac8 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -36,7 +36,7 @@ KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java index 82494f2f0d..08ca0ab511 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java @@ -45,33 +45,69 @@ class Chunker implements Iterator, Iterable { private static final Charset UTF_8 = StandardCharsets.UTF_8; //Chunking algorithm paramaters-------------------------------------// - /** the maximum size of a chunk, including the window. */ + /** + * the maximum size of a chunk, including the window. + */ private static final int MAX_TOTAL_CHUNK_SIZE = 32760; //bytes - /** the minimum to read before we start the process of looking for - * whitespace to break at and creating an overlapping window. */ + /** + * the minimum to read before we start the process of looking for whitespace + * to break at and creating an overlapping window. + */ private static final int MINIMUM_BASE_CHUNK_SIZE = 30 * 1024; //bytes - /** The maximum size of the chunk, before the overlapping window, even if we - * couldn't find whitespace to break at. */ + /** + * The maximum size of the chunk, before the overlapping window, even if we + * couldn't find whitespace to break at. + */ private static final int MAXIMUM_BASE_CHUNK_SIZE = 31 * 1024; //bytes - /** The amount of text we will read through before we give up on finding - * whitespace to break the chunk/window at. */ + /** + * The amount of text we will read through before we give up on finding + * whitespace to break the chunk/window at. + */ private static final int WHITE_SPACE_BUFFER_SIZE = 512; //bytes - /** The number of characters to read in one go from the Reader. */ + /** + * The number of characters to read in one go from the Reader. + */ private static final int READ_CHARS_BUFFER_SIZE = 512; //chars + /** + * When toLowerCase() is called on a character, the lower cased output + * can be different in size than the original input. I have seen a single + * input character turn into 3 characters (and 5 bytes) after lowercasing. + * I could not find any info as to what is the upper limit of how much a + * character can "increase in size" during lower casing. I'm guestimating + * and setting that limit at 10 bytes. + */ + private static final int MAX_CHAR_SIZE_INCREASE_IN_BYTES = 10; //bytes ////chunker state--------------------------------------------/// - /** The Reader that this chunk reads from, and divides into chunks. It must - * be a buffered reader to ensure that mark/reset are supported. */ + /** + * The Reader that this chunk reads from, and divides into chunks. It must + * be a buffered reader to ensure that mark/reset are supported. + */ private final PushbackReader reader; - /** The local buffer of characters read from the Reader. */ + /** + * The local buffer of characters read from the Reader. + */ private final char[] tempChunkBuf = new char[READ_CHARS_BUFFER_SIZE]; - /** the size in bytes of the chunk (so far). */ + /** + * the size in bytes of the chunk (so far). + */ private int chunkSizeBytes = 0; - /** Has the chunker reached the end of the Reader? If so, there are no more - * chunks, and the current chunk does not need a window. */ + + /** + * the size in bytes of the lowercased chunk (so far). Note that lowercasing + * in Java can change the size of the string so we need to make sure the + * lowercased string also fits in MAX_TOTAL_CHUNK_SIZE. + */ + private int lowerCasedChunkSizeBytes = 0; + /** + * Has the chunker reached the end of the Reader? If so, there are no more + * chunks, and the current chunk does not need a window. + */ private boolean endOfReaderReached = false; - /** Store any exception encountered reading from the Reader. */ + /** + * Store any exception encountered reading from the Reader. + */ private Exception ex; /** @@ -140,7 +176,7 @@ class Chunker implements Iterator, Iterable { * @param s The string to cleanup. * * @return A StringBuilder with the same content as s but where all invalid - * code * points have been replaced. + * code * points have been replaced. */ private static StringBuilder replaceInvalidUTF16(String s) { /* encode the string to UTF-16 which does the replcement, see @@ -162,16 +198,18 @@ class Chunker implements Iterator, Iterable { //reset state for the next chunk chunkSizeBytes = 0; + lowerCasedChunkSizeBytes = 0; int baseChunkSizeChars = 0; StringBuilder currentChunk = new StringBuilder(); StringBuilder currentWindow = new StringBuilder(); + StringBuilder lowerCasedChunk = new StringBuilder(); try { - currentChunk.append(readBaseChunk()); + readBaseChunk(currentChunk, lowerCasedChunk); baseChunkSizeChars = currentChunk.length(); //save the base chunk length - currentWindow.append(readWindow()); - //add the window text to the current chunk. - currentChunk.append(currentWindow); + readWindow(currentWindow, lowerCasedChunk); + //add the window text to the current chunk. + currentChunk.append(currentWindow); if (endOfReaderReached) { /* if we have reached the end of the content,we won't make * another overlapping chunk, so the length of the base chunk @@ -186,9 +224,9 @@ class Chunker implements Iterator, Iterable { * and break any chunking loop in client code. */ ex = ioEx; } - + //sanitize the text and return a Chunk object, that includes the base chunk length. - return new Chunk(currentChunk, baseChunkSizeChars, chunkSizeBytes); + return new Chunk(currentChunk, baseChunkSizeChars, lowerCasedChunk); } /** @@ -196,14 +234,12 @@ class Chunker implements Iterator, Iterable { * * @throws IOException if there is a problem reading from the reader. */ - private StringBuilder readBaseChunk() throws IOException { - StringBuilder currentChunk = new StringBuilder(); + private void readBaseChunk(StringBuilder currentChunk, StringBuilder lowerCasedChunk) throws IOException { //read the chunk until the minimum base chunk size - readHelper(MINIMUM_BASE_CHUNK_SIZE, currentChunk); + readHelper(MINIMUM_BASE_CHUNK_SIZE, currentChunk, lowerCasedChunk); //keep reading until the maximum base chunk size or white space is reached. - readToWhiteSpaceHelper(MAXIMUM_BASE_CHUNK_SIZE, currentChunk); - return currentChunk; + readToWhiteSpaceHelper(MAXIMUM_BASE_CHUNK_SIZE, currentChunk, lowerCasedChunk); } /** @@ -211,14 +247,12 @@ class Chunker implements Iterator, Iterable { * * @throws IOException if there is a problem reading from the reader. */ - private StringBuilder readWindow() throws IOException { - StringBuilder currentWindow = new StringBuilder(); + private void readWindow(StringBuilder currentChunk, StringBuilder lowerCasedChunk) throws IOException { //read the window, leaving some room to look for white space to break at. - readHelper(MAX_TOTAL_CHUNK_SIZE - WHITE_SPACE_BUFFER_SIZE, currentWindow); + readHelper(MAX_TOTAL_CHUNK_SIZE - WHITE_SPACE_BUFFER_SIZE, currentChunk, lowerCasedChunk); //keep reading until the max chunk size, or until whitespace is reached. - readToWhiteSpaceHelper(MAX_TOTAL_CHUNK_SIZE, currentWindow); - return currentWindow; + readToWhiteSpaceHelper(MAX_TOTAL_CHUNK_SIZE, currentChunk, lowerCasedChunk); } /** @@ -229,10 +263,10 @@ class Chunker implements Iterator, Iterable { * * @throws IOException */ - private void readHelper(int maxBytes, StringBuilder currentSegment) throws IOException { + private void readHelper(int maxBytes, StringBuilder currentSegment, StringBuilder currentLowerCasedSegment) throws IOException { int charsRead = 0; //read chars up to maxBytes, or the end of the reader. - while ((chunkSizeBytes < maxBytes) + while ((chunkSizeBytes < maxBytes) && (lowerCasedChunkSizeBytes < maxBytes) && (endOfReaderReached == false)) { charsRead = reader.read(tempChunkBuf, 0, READ_CHARS_BUFFER_SIZE); if (-1 == charsRead) { @@ -253,11 +287,19 @@ class Chunker implements Iterator, Iterable { //get the length in utf8 bytes of the read chars int segmentSize = chunkSegment.toString().getBytes(UTF_8).length; + // lower case the string and get it's size. NOTE: lower casing can + // change the size of the string! + String lowerCasedSegment = chunkSegment.toString().toLowerCase(); + int lowerCasedSegmentSize = lowerCasedSegment.getBytes(UTF_8).length; + //if it will not put us past maxBytes - if (chunkSizeBytes + segmentSize < maxBytes) { + if ((chunkSizeBytes + segmentSize < maxBytes) && (lowerCasedChunkSizeBytes + lowerCasedSegmentSize < maxBytes)) { //add it to the chunk currentSegment.append(chunkSegment); chunkSizeBytes += segmentSize; + + currentLowerCasedSegment.append(lowerCasedSegment); + lowerCasedChunkSizeBytes += lowerCasedSegmentSize; } else { //unread it, and break out of read loop. reader.unread(tempChunkBuf, 0, charsRead); @@ -275,11 +317,12 @@ class Chunker implements Iterator, Iterable { * * @throws IOException */ - private void readToWhiteSpaceHelper(int maxBytes, StringBuilder currentChunk) throws IOException { + private void readToWhiteSpaceHelper(int maxBytes, StringBuilder currentChunk, StringBuilder lowerCasedChunk) throws IOException { int charsRead = 0; boolean whitespaceFound = false; //read 1 char at a time up to maxBytes, whitespaceFound, or we reach the end of the reader. - while ((chunkSizeBytes < maxBytes) + while ((chunkSizeBytes < maxBytes - MAX_CHAR_SIZE_INCREASE_IN_BYTES) + && (lowerCasedChunkSizeBytes < maxBytes - MAX_CHAR_SIZE_INCREASE_IN_BYTES) && (whitespaceFound == false) && (endOfReaderReached == false)) { charsRead = reader.read(tempChunkBuf, 0, 1); @@ -314,6 +357,12 @@ class Chunker implements Iterator, Iterable { //add read chars to the chunk and update the length. currentChunk.append(sanitizedChunkSegment); chunkSizeBytes += sanitizedChunkSegment.toString().getBytes(UTF_8).length; + + // lower case the string and get it's size. NOTE: lower casing can + // change the size of the string. + String lowerCasedSegment = sanitizedChunkSegment.toString().toLowerCase(); + lowerCasedChunk.append(lowerCasedSegment); + lowerCasedChunkSizeBytes += lowerCasedSegment.getBytes(UTF_8).length; } } } @@ -326,16 +375,16 @@ class Chunker implements Iterator, Iterable { private final StringBuilder sb; private final int baseChunkSizeChars; - private final int chunkSizeBytes; + private final StringBuilder lowerCasedChunk; - Chunk(StringBuilder sb, int baseChunkSizeChars, int chunkSizeBytes) { + Chunk(StringBuilder sb, int baseChunkSizeChars, StringBuilder lowerCasedChunk) { this.sb = sb; this.baseChunkSizeChars = baseChunkSizeChars; - this.chunkSizeBytes = chunkSizeBytes; + this.lowerCasedChunk = lowerCasedChunk; } /** - * Get the content of the chunk. + * Get the content of the original (non-lower cased) chunk. * * @return The content of the chunk. */ @@ -345,16 +394,16 @@ class Chunker implements Iterator, Iterable { } /** - * Get the size in bytes of the utf-8 encoding of the entire chunk. + * Get the content of the lower cased chunk. * - * @return the size in bytes of the utf-8 encoding of the entire chunk + * @return The content of the chunk. */ - public int getChunkSizeBytes() { - return chunkSizeBytes; + public String geLowerCasedChunk() { + return lowerCasedChunk.toString(); } /** - * Get the length of the base chunk in java chars. + * Get the length of the original (non-lower cased) base chunk in java chars. * * @return the length of the base chunk in java chars. */ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java index 240c10e431..1eb30f5b6c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java @@ -38,6 +38,7 @@ import org.apache.commons.lang3.math.NumberUtils; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest.METHOD; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; @@ -346,6 +347,8 @@ class HighlightedText implements IndexedText { String chunkID = ""; String highlightField = ""; try { + double indexSchemaVersion = NumberUtils.toDouble(solrServer.getIndexInfo().getSchemaVersion()); + loadPageInfo(); //inits once SolrQuery q = new SolrQuery(); q.setShowDebugInfo(DEBUG); //debug @@ -359,22 +362,46 @@ class HighlightedText implements IndexedText { highlightField = LuceneQuery.HIGHLIGHT_FIELD; if (isLiteral) { - //if the query is literal try to get solr to do the highlighting - final String highlightQuery = keywords.stream() - .map(HighlightedText::constructEscapedSolrQuery) - .collect(Collectors.joining(" ")); + if (2.2 <= indexSchemaVersion) { + //if the query is literal try to get solr to do the highlighting + final String highlightQuery = keywords.stream().map(s -> + LanguageSpecificContentQueryHelper.expandQueryString(KeywordSearchUtil.quoteQuery(KeywordSearchUtil.escapeLuceneQuery(s)))) + .collect(Collectors.joining(" OR ")); + q.setQuery(highlightQuery); + for (Server.Schema field : LanguageSpecificContentQueryHelper.getQueryFields()) { + q.addField(field.toString()); + q.addHighlightField(field.toString()); + } + q.addField(Server.Schema.LANGUAGE.toString()); + // in case of single term literal query there is only 1 term + LanguageSpecificContentQueryHelper.configureTermfreqQuery(q, keywords.iterator().next()); + q.addFilterQuery(filterQuery); + q.setHighlightFragsize(0); // don't fragment the highlight, works with original highlighter, or needs "single" list builder with FVH + } else { + //if the query is literal try to get solr to do the highlighting + final String highlightQuery = keywords.stream() + .map(HighlightedText::constructEscapedSolrQuery) + .collect(Collectors.joining(" ")); - q.setQuery(highlightQuery); - q.addField(highlightField); - q.addFilterQuery(filterQuery); - q.addHighlightField(highlightField); - q.setHighlightFragsize(0); // don't fragment the highlight, works with original highlighter, or needs "single" list builder with FVH + q.setQuery(highlightQuery); + q.addField(highlightField); + q.addFilterQuery(filterQuery); + q.addHighlightField(highlightField); + q.setHighlightFragsize(0); // don't fragment the highlight, works with original highlighter, or needs "single" list builder with FVH + } //tune the highlighter - q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one NON-NLS - q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only NON-NLS - q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only NON-NLS - q.setParam("hl.fragListBuilder", "single"); //makes sense for FastVectorHighlighter only NON-NLS + if (shouldUseOriginalHighlighter(filterQuery)) { + // use original highlighter + q.setParam("hl.useFastVectorHighlighter", "off"); + q.setParam("hl.simple.pre", HIGHLIGHT_PRE); + q.setParam("hl.simple.post", HIGHLIGHT_POST); + } else { + q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one NON-NLS + q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only NON-NLS + q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only NON-NLS + q.setParam("hl.fragListBuilder", "single"); //makes sense for FastVectorHighlighter only NON-NLS + } //docs says makes sense for the original Highlighter only, but not really q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //NON-NLS @@ -406,12 +433,40 @@ class HighlightedText implements IndexedText { if (responseHighlightID == null) { highlightedContent = attemptManualHighlighting(response.getResults(), highlightField, keywords); } else { - List contentHighlights = responseHighlightID.get(LuceneQuery.HIGHLIGHT_FIELD); - if (contentHighlights == null) { - highlightedContent = attemptManualHighlighting(response.getResults(), highlightField, keywords); + SolrDocument document = response.getResults().get(0); + Object language = document.getFieldValue(Server.Schema.LANGUAGE.toString()); + if (2.2 <= indexSchemaVersion && language != null) { + List contentHighlights = LanguageSpecificContentQueryHelper.getHighlights(responseHighlightID).orElse(null); + if (contentHighlights == null) { + highlightedContent = ""; + } else { + int hitCountInMiniChunk = LanguageSpecificContentQueryHelper.queryChunkTermfreq(keywords, MiniChunkHelper.getChunkIdString(contentIdStr)); + String s = contentHighlights.get(0).trim(); + // If there is a mini-chunk, trim the content not to show highlighted text in it. + if (0 < hitCountInMiniChunk) { + int hitCountInChunk = ((Float) document.getFieldValue(Server.Schema.TERMFREQ.toString())).intValue(); + int idx = LanguageSpecificContentQueryHelper.findNthIndexOf( + s, + HIGHLIGHT_PRE, + // trim after the last hit in chunk + hitCountInChunk - hitCountInMiniChunk); + if (idx != -1) { + highlightedContent = s.substring(0, idx); + } else { + highlightedContent = s; + } + } else { + highlightedContent = s; + } + } } else { - // extracted content (minus highlight tags) is HTML-escaped - highlightedContent = contentHighlights.get(0).trim(); + List contentHighlights = responseHighlightID.get(LuceneQuery.HIGHLIGHT_FIELD); + if (contentHighlights == null) { + highlightedContent = attemptManualHighlighting(response.getResults(), highlightField, keywords); + } else { + // extracted content (minus highlight tags) is HTML-escaped + highlightedContent = contentHighlights.get(0).trim(); + } } } } @@ -551,4 +606,39 @@ class HighlightedText implements IndexedText { return buf.toString(); } + /** + * Return true if we should use original highlighter instead of FastVectorHighlighter. + * + * In the case Japanese text and phrase query, FastVectorHighlighter does not work well. + * + * Note about highlighters: + * If the query is "雨が降る" (phrase query), Solr divides it into 雨 and 降る. が is a stop word here. + * It seems that FastVector highlighter does not produce any snippet when there is a stop word between terms. + * On the other hand, original highlighter produces multiple matches, for example: + * > 雨が降っています + * Unified highlighter (from Solr 6.4) handles the case as expected: + * > 雨が降っています。 + * + * @param filterQuery An already properly escaped filter query. + */ + private boolean shouldUseOriginalHighlighter(String filterQuery) throws NoOpenCoreException, KeywordSearchModuleException { + final SolrQuery q = new SolrQuery(); + q.setQuery("*:*"); + q.addFilterQuery(filterQuery); + q.setFields(Server.Schema.LANGUAGE.toString()); + + QueryResponse response = solrServer.query(q, METHOD.POST); + SolrDocumentList solrDocuments = response.getResults(); + + if (!solrDocuments.isEmpty()) { + SolrDocument solrDocument = solrDocuments.get(0); + if (solrDocument != null) { + Object languageField = solrDocument.getFieldValue(Server.Schema.LANGUAGE.toString()); + if (languageField != null) { + return languageField.equals("ja"); + } + } + } + return false; + } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index e46791d270..e2abde6eb0 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -39,7 +39,7 @@ class IndexFinder { private static final String KWS_DATA_FOLDER_NAME = "data"; private static final String INDEX_FOLDER_NAME = "index"; private static final String CURRENT_SOLR_VERSION = "4"; - private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.1"; + private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.2"; static String getCurrentSolrVersion() { return CURRENT_SOLR_VERSION; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index bcdf143697..576b65d581 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -20,8 +20,10 @@ package org.sleuthkit.autopsy.keywordsearch; import java.io.BufferedReader; import java.io.Reader; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import org.apache.commons.lang3.math.NumberUtils; import org.apache.solr.client.solrj.SolrServerException; @@ -59,6 +61,8 @@ class Ingester { private final Server solrServer = KeywordSearch.getServer(); private static final SolrFieldsVisitor SOLR_FIELDS_VISITOR = new SolrFieldsVisitor(); private static Ingester instance; + private final LanguageSpecificContentIndexingHelper languageSpecificContentIndexingHelper + = new LanguageSpecificContentIndexingHelper(); private Ingester() { } @@ -93,7 +97,7 @@ class Ingester { * file, but the Solr server is probably fine. */ void indexMetaDataOnly(AbstractFile file) throws IngesterException { - indexChunk("", file.getName().toLowerCase(), getContentFields(file)); + indexChunk("", "", file.getName().toLowerCase(), new HashMap<>(getContentFields(file))); } /** @@ -107,7 +111,7 @@ class Ingester { * artifact, but the Solr server is probably fine. */ void indexMetaDataOnly(BlackboardArtifact artifact, String sourceName) throws IngesterException { - indexChunk("", sourceName, getContentFields(artifact)); + indexChunk("", "", sourceName, new HashMap<>(getContentFields(artifact))); } /** @@ -143,21 +147,30 @@ class Ingester { < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException { int numChunks = 0; //unknown until chunking is done - Map fields = getContentFields(source); + Map contentFields = Collections.unmodifiableMap(getContentFields(source)); //Get a reader for the content of the given source try (BufferedReader reader = new BufferedReader(sourceReader)) { Chunker chunker = new Chunker(reader); - for (Chunk chunk : chunker) { + while (chunker.hasNext()) { if (context != null && context.fileIngestIsCancelled()) { logger.log(Level.INFO, "File ingest cancelled. Cancelling keyword search indexing of {0}", sourceName); return false; } + + Chunk chunk = chunker.next(); + Map fields = new HashMap<>(contentFields); String chunkId = Server.getChunkIdString(sourceID, numChunks + 1); fields.put(Server.Schema.ID.toString(), chunkId); fields.put(Server.Schema.CHUNK_SIZE.toString(), String.valueOf(chunk.getBaseChunkLength())); + Optional language = languageSpecificContentIndexingHelper.detectLanguageIfNeeded(chunk); + language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang)); try { //add the chunk text to Solr index - indexChunk(chunk.toString(), sourceName, fields); + indexChunk(chunk.toString(), chunk.geLowerCasedChunk(), sourceName, fields); + // add mini chunk when there's a language specific field + if (chunker.hasNext() && language.isPresent()) { + languageSpecificContentIndexingHelper.indexMiniChunk(chunk, sourceName, new HashMap<>(contentFields), chunkId, language.get()); + } numChunks++; } catch (Ingester.IngesterException ingEx) { logger.log(Level.WARNING, "Ingester had a problem with extracted string from file '" //NON-NLS @@ -177,13 +190,14 @@ class Ingester { if (context != null && context.fileIngestIsCancelled()) { return false; } else { + Map fields = new HashMap<>(contentFields); //after all chunks, index just the meta data, including the numChunks, of the parent file fields.put(Server.Schema.NUM_CHUNKS.toString(), Integer.toString(numChunks)); //reset id field to base document id fields.put(Server.Schema.ID.toString(), Long.toString(sourceID)); //"parent" docs don't have chunk_size fields.remove(Server.Schema.CHUNK_SIZE.toString()); - indexChunk(null, sourceName, fields); + indexChunk(null, null, sourceName, fields); } } return true; @@ -197,12 +211,13 @@ class Ingester { * 4.0.0), see if possible to stream with UpdateRequestHandler * * @param chunk The chunk content as a string, or null for metadata only + * @param lowerCasedChunk The lower cased chunk content as a string, or null for metadata only * @param fields * @param size * * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException */ - private void indexChunk(String chunk, String sourceName, Map fields) throws IngesterException { + private void indexChunk(String chunk, String lowerCasedChunk, String sourceName, Map fields) throws IngesterException { if (fields.get(Server.Schema.IMAGE_ID.toString()) == null) { //JMTODO: actually if the we couldn't get the image id it is set to -1, // but does this really mean we don't want to index it? @@ -231,7 +246,7 @@ class Ingester { // insensitive substring/regular expression search. double indexSchemaVersion = NumberUtils.toDouble(solrServer.getIndexInfo().getSchemaVersion()); if (indexSchemaVersion >= 2.1) { - updateDoc.addField(Server.Schema.CONTENT_STR.toString(), ((chunk == null) ? "" : chunk.toLowerCase())); + updateDoc.addField(Server.Schema.CONTENT_STR.toString(), ((chunk == null) ? "" : lowerCasedChunk)); } TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Index chunk"); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Language.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Language.java new file mode 100755 index 0000000000..5fb1f859d3 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Language.java @@ -0,0 +1,46 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.Arrays; +import java.util.Optional; + +/** + * Language. + * + * Contents which are detected to have these languages should be indexed to a corresponding language-specific field + * such as content_ja. + */ +public enum Language { + JAPANESE("ja"); + + private String value; + + String getValue() { + return value; + } + + static Optional fromValue(String value) { + return Arrays.stream(Language.values()).filter(x -> x.value.equals(value)).findFirst(); + } + + Language(String value) { + this.value = value; + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageDetector.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageDetector.java new file mode 100755 index 0000000000..f527a2fc0e --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageDetector.java @@ -0,0 +1,60 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import com.optimaize.langdetect.LanguageDetectorBuilder; +import com.optimaize.langdetect.i18n.LdLocale; +import com.optimaize.langdetect.ngram.NgramExtractors; +import com.optimaize.langdetect.profiles.LanguageProfileReader; +import com.optimaize.langdetect.text.CommonTextObjectFactories; +import com.optimaize.langdetect.text.TextObject; +import com.optimaize.langdetect.text.TextObjectFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Optional; + +/** + * Detects the language of the given contents. Only languages which should be indexed to a corresponding + * language-specific field are detected. + */ +class LanguageDetector { + + private com.optimaize.langdetect.LanguageDetector impl; + private TextObjectFactory textObjectFactory; + + LanguageDetector() { + try { + impl = LanguageDetectorBuilder.create(NgramExtractors.standard()) + .withProfiles(new LanguageProfileReader().readAllBuiltIn()) + .build(); + textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); + } catch (IOException e) { + // The IOException here could occur when failing to read the language profiles from the classpath. + // That can be considered to be a severe IO problem. Nothing can be done here. + throw new UncheckedIOException(e); + } + } + + Optional detect(String text) { + TextObject textObject = textObjectFactory.forText(text); + Optional localeOpt = impl.detect(textObject).transform(Optional::of).or(Optional.empty()); + return localeOpt.map(LdLocale::getLanguage).flatMap(Language::fromValue); + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentIndexingHelper.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentIndexingHelper.java new file mode 100755 index 0000000000..d0988c83f3 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentIndexingHelper.java @@ -0,0 +1,85 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.solr.common.SolrInputDocument; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.healthmonitor.HealthMonitor; +import org.sleuthkit.autopsy.healthmonitor.TimingMetric; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A helper class to support indexing language-specific fields. + */ +class LanguageSpecificContentIndexingHelper { + + private final LanguageDetector languageDetector = new LanguageDetector(); + + Optional detectLanguageIfNeeded(Chunker.Chunk chunk) throws NoOpenCoreException { + double indexSchemaVersion = NumberUtils.toDouble(KeywordSearch.getServer().getIndexInfo().getSchemaVersion()); + if (2.2 <= indexSchemaVersion) { + return languageDetector.detect(chunk.toString()); + } else { + return Optional.empty(); + } + } + + void updateLanguageSpecificFields(Map fields, Chunker.Chunk chunk, Language language) { + List values = new ArrayList<>(); + values.add(chunk.toString()); + if (fields.containsKey(Server.Schema.FILE_NAME.toString())) { + values.add(fields.get(Server.Schema.FILE_NAME.toString()).toString()); + } + + // index the chunk to a language specific field + fields.put(Server.Schema.CONTENT_JA.toString(), values); + fields.put(Server.Schema.LANGUAGE.toString(), language.getValue()); + } + + void indexMiniChunk(Chunker.Chunk chunk, String sourceName, Map fields, String baseChunkID, Language language) + throws Ingester.IngesterException { + //Make a SolrInputDocument out of the field map + SolrInputDocument updateDoc = new SolrInputDocument(); + for (String key : fields.keySet()) { + updateDoc.addField(key, fields.get(key)); + } + + try { + updateDoc.setField(Server.Schema.ID.toString(), MiniChunkHelper.getChunkIdString(baseChunkID)); + + // index the chunk to a language specific field + updateDoc.addField(Server.Schema.CONTENT_JA.toString(), chunk.toString().substring(chunk.getBaseChunkLength())); + updateDoc.addField(Server.Schema.LANGUAGE.toString(), language.getValue()); + + TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Index chunk"); + + KeywordSearch.getServer().addDocument(updateDoc); + HealthMonitor.submitTimingMetric(metric); + + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + throw new Ingester.IngesterException( + NbBundle.getMessage(Ingester.class, "Ingester.ingest.exception.err.msg", sourceName), ex); + } + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentQueryHelper.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentQueryHelper.java new file mode 100755 index 0000000000..a3ed8a7876 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentQueryHelper.java @@ -0,0 +1,248 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.sleuthkit.autopsy.coreutils.EscapeUtil; +import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.datamodel.TskException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A helper class to support querying documents which have language-specific fields. + */ +final class LanguageSpecificContentQueryHelper { + + private LanguageSpecificContentQueryHelper() {} + + private static final List QUERY_FIELDS = new ArrayList<>(); + private static final List LANGUAGE_SPECIFIC_CONTENT_FIELDS + = Collections.singletonList(Server.Schema.CONTENT_JA); + private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT); + + static { + QUERY_FIELDS.add(Server.Schema.TEXT); + QUERY_FIELDS.addAll(LANGUAGE_SPECIFIC_CONTENT_FIELDS); + } + + /** + * Holds query response for later processes related to language-specific fields + */ + static class QueryResults { + List chunks = new ArrayList<>(); + Map miniChunks = new HashMap<>(); + // objectId_chunk -> "text" -> List of previews + Map>> highlighting = new HashMap<>(); + } + + /** + * Make a query string from the given one by applying it to the multiple query fields + * + * @param queryStr escaped query string + * @return query string + */ + static String expandQueryString(final String queryStr) { + List fieldQueries = new ArrayList<>(); + fieldQueries.add(Server.Schema.TEXT.toString() + ":" + queryStr); + fieldQueries.addAll(LANGUAGE_SPECIFIC_CONTENT_FIELDS.stream().map(field -> field.toString() + ":" + queryStr).collect(Collectors.toList())); + return String.join(" OR ", fieldQueries); + } + + static List getQueryFields() { + return QUERY_FIELDS; + } + + static void updateQueryResults(QueryResults results, SolrDocument document) { + String id = (String) document.getFieldValue(Server.Schema.ID.toString()); + if (MiniChunkHelper.isMiniChunkID(id)) { + results.miniChunks.put(MiniChunkHelper.getBaseChunkID(id), document); + } else { + results.chunks.add(document); + } + } + + /** + * Get snippets + * + * @param highlight field ID -> snippets + * @return snippets of appropriate fields. + * Note that this method returns {@code Optional.empty} if the result is empty for convenience to interact with the existing code. + */ + static Optional> getHighlights(Map> highlight) { + for (Server.Schema field : LANGUAGE_SPECIFIC_CONTENT_FIELDS) { + if (highlight.containsKey(field.toString())) { + return Optional.of(highlight.get(field.toString())); + } + } + return Optional.empty(); + } + + /** + * Merge KeywordHits from TEXT field and a language specific field + * + * Replace KeywordHits in the given {@code matches} if its chunk ID is same. + */ + static List mergeKeywordHits(List matches, Keyword originalKeyword, QueryResults queryResults) throws KeywordSearchModuleException { + Map map = findMatches(originalKeyword, queryResults).stream().collect(Collectors.toMap(KeywordHit::getSolrDocumentId, x -> x)); + List merged = new ArrayList<>(); + + // first, replace KeywordHit in matches + for (KeywordHit match : matches) { + String key = match.getSolrDocumentId(); + if (map.containsKey(key)) { + merged.add(map.get(key)); + map.remove(key); + } else { + merged.add(match); + } + } + // second, add rest of KeywordHits from queryResults + merged.addAll(map.values()); + + return merged; + } + + static void configureTermfreqQuery(SolrQuery query, String keyword) throws KeywordSearchModuleException, NoOpenCoreException { + // make a request to Solr to parse query. + QueryTermHelper.Result queryParserResult = QueryTermHelper.parse(keyword, LANGUAGE_SPECIFIC_CONTENT_FIELDS); + query.addField(buildTermfreqQuery(keyword, queryParserResult)); + } + + static String buildTermfreqQuery(String keyword, QueryTermHelper.Result result) { + List termfreqs = new ArrayList<>(); + for (Map.Entry> e : result.fieldTermsMap.entrySet()) { + String field = e.getKey(); + for (String term : e.getValue()) { + termfreqs.add(String.format("termfreq(\"%s\",\"%s\")", field, KeywordSearchUtil.escapeLuceneQuery(term))); + } + } + + // sum of all language specific query fields. + // only one of these fields could be non-zero. + return String.format("termfreq:sum(%s)", String.join(",", termfreqs)); + } + + static int queryChunkTermfreq(Set keywords, String contentID) throws KeywordSearchModuleException, NoOpenCoreException { + SolrQuery q = new SolrQuery(); + q.setShowDebugInfo(DEBUG); + + final String filterQuery = Server.Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(contentID); + final String highlightQuery = keywords.stream() + .map(s -> LanguageSpecificContentQueryHelper.expandQueryString( + KeywordSearchUtil.quoteQuery(KeywordSearchUtil.escapeLuceneQuery(s)))) + .collect(Collectors.joining(" ")); + + q.addFilterQuery(filterQuery); + q.setQuery(highlightQuery); + LanguageSpecificContentQueryHelper.configureTermfreqQuery(q, keywords.iterator().next()); + + QueryResponse response = KeywordSearch.getServer().query(q, SolrRequest.METHOD.POST); + SolrDocumentList results = response.getResults(); + if (results.isEmpty()) { + return 0; + } + + SolrDocument document = results.get(0); + return ((Float) document.getFieldValue(Server.Schema.TERMFREQ.toString())).intValue(); + } + + static int findNthIndexOf(String s, String pattern, int n) { + int found = 0; + int idx = -1; + int len = s.length(); + while (idx < len && found <= n) { + idx = s.indexOf(pattern, idx + 1); + if (idx == -1) { + break; + } + found++; + } + + return idx; + } + + private static List findMatches(Keyword originalKeyword, QueryResults queryResults) throws KeywordSearchModuleException { + List matches = new ArrayList<>(); + for (SolrDocument document : queryResults.chunks) { + String docId = (String) document.getFieldValue(Server.Schema.ID.toString()); + + try { + int hitCountInChunk = ((Float) document.getFieldValue(Server.Schema.TERMFREQ.toString())).intValue(); + SolrDocument miniChunk = queryResults.miniChunks.get(docId); + if (miniChunk == null) { + // last chunk does not have mini chunk because there's no overlapped region with next one + matches.add(createKeywordHit(originalKeyword, queryResults.highlighting, docId)); + } else { + int hitCountInMiniChunk = ((Float) miniChunk.getFieldValue(Server.Schema.TERMFREQ.toString())).intValue(); + if (hitCountInMiniChunk < hitCountInChunk) { + // there are at least one hit in base chunk + matches.add(createKeywordHit(originalKeyword, queryResults.highlighting, docId)); + } + } + } catch (TskException ex) { + throw new KeywordSearchModuleException(ex); + } + } + return matches; + } + + /** + * copied from LuceneQuery and modified to use getHighlightFieldValue + */ + private static KeywordHit createKeywordHit(Keyword originalKeyword, Map>> highlightResponse, String docId) throws TskException { + /** + * Get the first snippet from the document if keyword search is + * configured to use snippets. + */ + String snippet = ""; + if (KeywordSearchSettings.getShowSnippets()) { + List snippetList = getHighlightFieldValue(highlightResponse.get(docId)).orElse(null); + // list is null if there wasn't a snippet + if (snippetList != null) { + snippet = EscapeUtil.unEscapeHtml(snippetList.get(0)).trim(); + } + } + + return new KeywordHit(docId, snippet, originalKeyword.getSearchTerm()); + } + + /** + * @return Optional.empty if empty + */ + private static Optional> getHighlightFieldValue(Map> highlight) { + for (Server.Schema field : LANGUAGE_SPECIFIC_CONTENT_FIELDS) { + if (highlight.containsKey(field.toString())) { + return Optional.of(highlight.get(field.toString())); + } + } + return Optional.empty(); + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 70c6155d5f..a324c03324 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -134,6 +134,7 @@ class LuceneQuery implements KeywordSearchQuery { String cursorMark = CursorMarkParams.CURSOR_MARK_START; boolean allResultsProcessed = false; List matches = new ArrayList<>(); + LanguageSpecificContentQueryHelper.QueryResults languageSpecificQueryResults = new LanguageSpecificContentQueryHelper.QueryResults(); while (!allResultsProcessed) { solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse response = solrServer.query(solrQuery, SolrRequest.METHOD.POST); @@ -141,7 +142,18 @@ class LuceneQuery implements KeywordSearchQuery { // objectId_chunk -> "text" -> List of previews Map>> highlightResponse = response.getHighlighting(); + if (2.2 <= indexSchemaVersion) { + languageSpecificQueryResults.highlighting.putAll(response.getHighlighting()); + } + for (SolrDocument resultDoc : resultList) { + if (2.2 <= indexSchemaVersion) { + Object language = resultDoc.getFieldValue(Server.Schema.LANGUAGE.toString()); + if (language != null) { + LanguageSpecificContentQueryHelper.updateQueryResults(languageSpecificQueryResults, resultDoc); + } + } + try { /* * for each result doc, check that the first occurence of @@ -153,6 +165,11 @@ class LuceneQuery implements KeywordSearchQuery { final Integer chunkSize = (Integer) resultDoc.getFieldValue(Server.Schema.CHUNK_SIZE.toString()); final Collection content = resultDoc.getFieldValues(Server.Schema.CONTENT_STR.toString()); + // if the document has language, it should be hit in language specific content fields. So skip here. + if (resultDoc.containsKey(Server.Schema.LANGUAGE.toString())) { + continue; + } + if (indexSchemaVersion < 2.0) { //old schema versions don't support chunk_size or the content_str fields, so just accept hits matches.add(createKeywordtHit(highlightResponse, docId)); @@ -179,9 +196,16 @@ class LuceneQuery implements KeywordSearchQuery { cursorMark = nextCursorMark; } + List mergedMatches; + if (2.2 <= indexSchemaVersion) { + mergedMatches = LanguageSpecificContentQueryHelper.mergeKeywordHits(matches, originalKeyword, languageSpecificQueryResults); + } else { + mergedMatches = matches; + } + QueryResults results = new QueryResults(this); //in case of single term literal query there is only 1 term - results.addResult(new Keyword(originalKeyword.getSearchTerm(), true, true, originalKeyword.getListName(), originalKeyword.getOriginalTerm()), matches); + results.addResult(new Keyword(originalKeyword.getSearchTerm(), true, true, originalKeyword.getListName(), originalKeyword.getOriginalTerm()), mergedMatches); return results; } @@ -262,19 +286,25 @@ class LuceneQuery implements KeywordSearchQuery { * * @return */ - private SolrQuery createAndConfigureSolrQuery(boolean snippets) { + private SolrQuery createAndConfigureSolrQuery(boolean snippets) throws NoOpenCoreException, KeywordSearchModuleException { + double indexSchemaVersion = NumberUtils.toDouble(KeywordSearch.getServer().getIndexInfo().getSchemaVersion()); + SolrQuery q = new SolrQuery(); q.setShowDebugInfo(DEBUG); //debug // Wrap the query string in quotes if this is a literal search term. String queryStr = originalKeyword.searchTermIsLiteral() - ? KeywordSearchUtil.quoteQuery(keywordStringEscaped) : keywordStringEscaped; + ? KeywordSearchUtil.quoteQuery(keywordStringEscaped) : keywordStringEscaped; // Run the query against an optional alternative field. if (field != null) { //use the optional field queryStr = field + ":" + queryStr; + q.setQuery(queryStr); + } else if (2.2 <= indexSchemaVersion && originalKeyword.searchTermIsLiteral()) { + q.setQuery(LanguageSpecificContentQueryHelper.expandQueryString(queryStr)); + } else { + q.setQuery(queryStr); } - q.setQuery(queryStr); q.setRows(MAX_RESULTS_PER_CURSOR_MARK); // Setting the sort order is necessary for cursor based paging to work. q.setSort(SolrQuery.SortClause.asc(Server.Schema.ID.toString())); @@ -283,6 +313,11 @@ class LuceneQuery implements KeywordSearchQuery { Server.Schema.CHUNK_SIZE.toString(), Server.Schema.CONTENT_STR.toString()); + if (2.2 <= indexSchemaVersion && originalKeyword.searchTermIsLiteral()) { + q.addField(Server.Schema.LANGUAGE.toString()); + LanguageSpecificContentQueryHelper.configureTermfreqQuery(q, keywordStringEscaped); + } + for (KeywordQueryFilter filter : filters) { q.addFilterQuery(filter.toString()); } @@ -300,8 +335,16 @@ class LuceneQuery implements KeywordSearchQuery { * * @param q The SolrQuery to configure. */ - private static void configurwQueryForHighlighting(SolrQuery q) { - q.addHighlightField(HIGHLIGHT_FIELD); + private static void configurwQueryForHighlighting(SolrQuery q) throws NoOpenCoreException { + double indexSchemaVersion = NumberUtils.toDouble(KeywordSearch.getServer().getIndexInfo().getSchemaVersion()); + if (2.2 <= indexSchemaVersion) { + for (Server.Schema field : LanguageSpecificContentQueryHelper.getQueryFields()) { + q.addHighlightField(field.toString()); + } + } else { + q.addHighlightField(HIGHLIGHT_FIELD); + } + q.setHighlightSnippets(1); q.setHighlightFragsize(SNIPPET_LENGTH); @@ -404,7 +447,13 @@ class LuceneQuery implements KeywordSearchQuery { if (responseHighlightID == null) { return ""; } - List contentHighlights = responseHighlightID.get(LuceneQuery.HIGHLIGHT_FIELD); + double indexSchemaVersion = NumberUtils.toDouble(solrServer.getIndexInfo().getSchemaVersion()); + List contentHighlights; + if (2.2 <= indexSchemaVersion) { + contentHighlights = LanguageSpecificContentQueryHelper.getHighlights(responseHighlightID).orElse(null); + } else { + contentHighlights = responseHighlightID.get(LuceneQuery.HIGHLIGHT_FIELD); + } if (contentHighlights == null) { return ""; } else { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/MiniChunkHelper.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/MiniChunkHelper.java new file mode 100755 index 0000000000..9e958587cd --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/MiniChunkHelper.java @@ -0,0 +1,41 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +/** + * Mini-chunk related methods. + */ +final class MiniChunkHelper { + + private MiniChunkHelper() {} + + static String SUFFIX = "_mini"; + + static String getChunkIdString(String baseChunkID) { + return baseChunkID + SUFFIX; + } + + static boolean isMiniChunkID(String chunkID) { + return chunkID.endsWith(SUFFIX); + } + + static String getBaseChunkID(String miniChunkID) { + return miniChunkID.replaceFirst(SUFFIX + "$", ""); + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryTermHelper.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryTermHelper.java new file mode 100755 index 0000000000..39a050c47f --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryTermHelper.java @@ -0,0 +1,95 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.FieldAnalysisRequest; +import org.apache.solr.client.solrj.response.AnalysisResponseBase; +import org.apache.solr.client.solrj.response.FieldAnalysisResponse; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Get terms from query using Solr. + * + * This class is used to find matched terms from query results. + */ +final class QueryTermHelper { + + private QueryTermHelper() {} + + /** + * Result of {@link #parse} method + */ + static class Result { + /** + * field name -> [term] + */ + final Map> fieldTermsMap = new HashMap<>(); + } + + /** + * Parse the given query string on Solr and return the result + * + * @param query query to parse + * @param fields field names to use for parsing + */ + static Result parse(String query, List fields) throws KeywordSearchModuleException, NoOpenCoreException { + Server server = KeywordSearch.getServer(); + + FieldAnalysisRequest request = new FieldAnalysisRequest(); + for (Server.Schema field : fields) { + request.addFieldName(field.toString()); + } + // FieldAnalysisRequest requires to set its field value property, + // while the corresponding analysis.fieldvalue parameter is not needed in the API. + // Setting an empty value does not effect on the result. + request.setFieldValue(""); + request.setQuery(query); + + FieldAnalysisResponse response = new FieldAnalysisResponse(); + try { + response.setResponse(server.request(request)); + } catch (SolrServerException e) { + throw new KeywordSearchModuleException(e); + } + + Result result = new Result(); + for (Map.Entry entry : response.getAllFieldNameAnalysis()) { + Iterator it = entry.getValue().getQueryPhases().iterator(); + + // The last phase is the one which is used in the search process. + AnalysisResponseBase.AnalysisPhase lastPhase = null; + while (it.hasNext()) { + lastPhase = it.next(); + } + + if (lastPhase != null) { + List tokens = lastPhase.getTokens().stream().map(AnalysisResponseBase.TokenInfo::getText).collect(Collectors.toList()); + result.fieldTermsMap.put(entry.getKey(), tokens); + } + } + + return result; + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 0e047456f6..a988417461 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -130,6 +130,18 @@ public class Server { return "content_ws"; //NON-NLS } }, + CONTENT_JA { + @Override + public String toString() { + return "content_ja"; //NON-NLS + } + }, + LANGUAGE { + @Override + public String toString() { + return "language"; //NON-NLS + } + }, FILE_NAME { @Override public String toString() { @@ -175,7 +187,18 @@ public class Server { public String toString() { return "chunk_size"; //NON-NLS } - } + }, + /** + * termfreq is a function which returns the number of times the term appears. + * This is not an actual field defined in schema.xml, but can be gotten from returned documents + * in the same way as fields. + */ + TERMFREQ { + @Override + public String toString() { + return "termfreq"; //NON-NLS + } + } }; public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented) diff --git a/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentQueryHelperTest.java b/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentQueryHelperTest.java new file mode 100755 index 0000000000..d8c876592e --- /dev/null +++ b/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/LanguageSpecificContentQueryHelperTest.java @@ -0,0 +1,59 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * tests for LanguageSpecificContentQueryHelper + */ +public class LanguageSpecificContentQueryHelperTest { + + @Test + public void makeQueryString() { + assertEquals("text:query OR content_ja:query", LanguageSpecificContentQueryHelper.expandQueryString("query")); + } + + @Test + public void findNthIndexOf() { + assertEquals(-1, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "_", 0)); + assertEquals(0, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "A", 0)); + assertEquals(2, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "A", 1)); + assertEquals(3, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "A", 2)); + assertEquals(-1, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "A", 3)); + assertEquals(0, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "", 0)); + assertEquals(-1, LanguageSpecificContentQueryHelper.findNthIndexOf("", "A", 0)); + assertEquals(-1, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "A", -1)); + assertEquals(-1, LanguageSpecificContentQueryHelper.findNthIndexOf("A1AA45", "A", 999)); + } + + @Test + public void buildTermfreqQuery() { + QueryTermHelper.Result result = new QueryTermHelper.Result(); + result.fieldTermsMap.put("field1", Arrays.asList("term1")); + result.fieldTermsMap.put("field2", Arrays.asList("term1", "term2")); + assertEquals( + "termfreq:sum(termfreq(\"field1\",\"term1\"),termfreq(\"field2\",\"term1\"),termfreq(\"field2\",\"term2\"))", + LanguageSpecificContentQueryHelper.buildTermfreqQuery("query", result)); + } +} diff --git a/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/MiniChunkHelperTest.java b/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/MiniChunkHelperTest.java new file mode 100755 index 0000000000..27336b8297 --- /dev/null +++ b/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/MiniChunkHelperTest.java @@ -0,0 +1,46 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * tests for MiniChunkHelper + */ +public class MiniChunkHelperTest { + + @Test + public void isMiniChunkID() { + assertTrue(MiniChunkHelper.isMiniChunkID("1_1_mini")); + assertFalse(MiniChunkHelper.isMiniChunkID("1_1")); + assertFalse(MiniChunkHelper.isMiniChunkID("1")); + } + + @Test + public void getBaseChunkID() { + Assert.assertEquals("1_1", MiniChunkHelper.getBaseChunkID("1_1_mini")); + Assert.assertEquals("1_1", MiniChunkHelper.getBaseChunkID("1_1")); + Assert.assertEquals("1", MiniChunkHelper.getBaseChunkID("1")); + } + +} \ No newline at end of file diff --git a/NEWS.txt b/NEWS.txt index b4bc6e8dbf..7a4487417d 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,3 +1,48 @@ +---------------- VERSION 4.13.0 -------------- +General: +- Switch from Oracle JDK to OpenJDK. +- Full command line support (case creation, adding of data sources, running ingest, and generating reports). + +Logical Imager: +- Output can be individual files instead of VHD image (uses less space). +- More fine grained progress during collection and importing. +- Log of files and make artifacts. +- All console messages are saved to a log file too. +- Improved handling of cancellation when adding results into a case. + +Ingest Modules: +- Added Android support as Python modules for: Android installed apps, Android browser, Facebook Messenger, IMO, LINE, Opera, ORUX Maps, Samsung SBrowser, Skype, ShareIt, TextNow, Viber, WhatsApp, Xender, Zapya. +- Recycle Bin files are parsed in Recent Activity module, new artifacts are created, and deleted file entries are created at the original location of the deleted files. Code is based on Mark McKinnon’s RecycleBin module (https://github.com/markmckinnon/Autopsy-Plugins/tree/master/Recycle_Bin). +- ShellBag registry data is extracted from RegRipper in the Recent Activity module. New artifacts are recreated for the data. Based on Mark McKinnon’s “Parse ShellBags” module (https://github.com/markmckinnon/Autopsy-Plugins/tree/master/Parse_Shellbags). +- Additional data is extracted about users from SAM hive in Recent Activity module. Data includes password dates, permissions, groups, and full name. Based on Mark McKinnon’s “Parse SAM” module (https://github.com/markmckinnon/Autopsy-Plugins/tree/master/Parse_SAM). +- Email ingest module parses EML files. Based on Mark McKinnon’s “EML Parser” module (https://github.com/markmckinnon/Autopsy-Plugins/tree/master/EML_Parser). +- Fixed bug in MBOX module that caused attachments to have a “_” in the name. +- New Plaso ingest module that runs Plaso and generates events for the timeline. +- Fixed bug in Email module for VCard files to better parse phone number types. +- Keyword Search module waits longer for Solr to start to prevent incorrectly reporting a problem and disabling the feature. +- Embedded file extractor module was updated to not report compression bombs for GZIP files. + +Timeline: +- New approach for storing event data. A dedicated events table exists and is populated as files and artifacts are added to the database. No longer requires an explicit step of populating a local events table. +- Users can create their own events from the Timeline UI. +- Filtering was simplified based or existence of tag or hash set hit versus a specific name. + +Communications: +- Fixed bug that hid contact book entries with duplicate numbers. + +Image Gallery: +- Fixed bug in schema that caused errors with very long file names. + +Report: +- CASE report is included in a portable case. +- Image tags are included in portable case. +- More size options for a packaged portable case. +- New Infrastructure to support command line-based generation. + +Backend: +- Developers should use new new Blackboard.postArtifact() method to ensure artifact is indexed and added to the timeline. +- New classes were created to make it easier to write modules for apps. + ---------------- VERSION 4.12.0 -------------- Collection - Added ability to configure a USB drive to use new logical imager tool. @@ -6,13 +51,13 @@ Collection Ingest Modules: - Changed file type detection so that Tika does not rely only on extension. -- Email ingest module assigns thread IDs to messages +- Email ingest module assigns thread IDs to messages. - Android ingest modules store thread ID from their databases. Content Viewers (lower right of UI): - New “Text” viewer that consolidates previous Strings and “Indexed Text” viewers. - New “Translation” panel was added to the new “Text” viewer. -- Added integration with Google and Bing translation (credentials required) +- Added integration with Google and Bing translation (credentials required). - Redesigned “Other Occurrences” viewer to have 4th column with details of selected item. - Added Willi Ballentin’s “Registry Hive Viewer” panel to the “Application” viewer. - Improved HTML viewer to use style sheets and better layout. diff --git a/README.txt b/README.txt index 73abd3e64e..3ed2dda963 100644 --- a/README.txt +++ b/README.txt @@ -72,11 +72,11 @@ GStreamer for viewing video files - Web page: http://gstreamer.freedesktop.org/ - License: http://www.gnu.org/licenses/lgpl.html -GStreamer-java for viewing video files -- Web page: http://code.google.com/p/gstreamer-java/ -- License: http://www.gnu.org/licenses/lgpl.html +GStreamer 1.x Java Core for viewing video files +- Web page: https://github.com/gstreamer-java/gst1-java-core +- License: https://github.com/gstreamer-java/gst1-java-core/blob/master/LICENSE.md -Regripper for pulling recently activity +Regripper for pulling recent activity (Including custom plugins) - Web page: http://regripper.wordpress.com/ - License: http://www.gnu.org/licenses/gpl.html diff --git a/RecentActivity/nbproject/project.xml b/RecentActivity/nbproject/project.xml index 269723962e..29b5d1362a 100644 --- a/RecentActivity/nbproject/project.xml +++ b/RecentActivity/nbproject/project.xml @@ -60,7 +60,7 @@ 10 - 10.16 + 10.17 @@ -69,7 +69,7 @@ 3 - 1.2 + 1.3 diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index f27b253e16..805e776717 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -2,9 +2,14 @@ cannotBuildXmlParser=Unable to build XML parser: cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml: cannotParseXml=Unable to parse XML file: ChromeCacheExtractor.moduleName=ChromeCacheExtractor +# {0} - module name +# {1} - row number +# {2} - table length +# {3} - cache path ChromeCacheExtractor.progressMsg={0}: Extracting cache entry {1} of {2} entries from {3} DataSourceUsage_AndroidMedia=Android Media Card DataSourceUsage_FlashDrive=Flash Drive +# {0} - OS name DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0}) DataSourceUsageAnalyzer.parentModuleName=Recent Activity Extract.indexError.message=Failed to index artifact for keyword search. @@ -46,6 +51,7 @@ ExtractOs.windowsVolume.label=OS Drive (Windows) ExtractOs.yellowDogLinuxOs.label=Linux (Yellow Dog) ExtractOs.yellowDogLinuxVolume.label=OS Drive (Linux Yellow Dog) ExtractOS_progressMessage=Checking for OS +ExtractRecycleBin_module_name=Recycle Bin ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files. ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files @@ -182,6 +188,7 @@ RecentDocumentsByLnk.parentModuleName.noSpace=RecentActivity RecentDocumentsByLnk.parentModuleName=Recent Activity RegRipperFullNotFound=Full version RegRipper executable not found. RegRipperNotFound=Autopsy RegRipper executable not found. +# {0} - file name SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}. SearchEngineURLQueryAnalyzer.moduleName.text=Search Engine SearchEngineURLQueryAnalyzer.engineName.none=NONE @@ -189,4 +196,7 @@ SearchEngineURLQueryAnalyzer.domainSubStr.none=NONE SearchEngineURLQueryAnalyzer.toString=Name: {0}\nDomain Substring: {1}\nCount: {2}\nSplit Tokens: \n{3} SearchEngineURLQueryAnalyzer.parentModuleName.noSpace=RecentActivity SearchEngineURLQueryAnalyzer.parentModuleName=Recent Activity +Shellbag_Artifact_Display_Name=Shell Bags +Shellbag_Key_Attribute_Display_Name=Key +Shellbag_Last_Write_Attribute_Display_Name=Last Write UsbDeviceIdMapper.parseAndLookup.text=Product: {0} diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java new file mode 100755 index 0000000000..d3af457a1f --- /dev/null +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -0,0 +1,620 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * + * Copyright 2012 42six Solutions. + * Contact: aebadirad 42six com + * Project Contact/Architect: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.recentactivity; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import org.joda.time.Instant; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.services.FileManager; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_OS_ACCOUNT; +import org.sleuthkit.datamodel.BlackboardAttribute; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_DELETED; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_ID; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.FsContent; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; +import org.sleuthkit.datamodel.TskDataException; + +/** + * This module is based on the RecycleBin python module from Mark McKinnon. + * + * @see + * Recycle_Bin.py + * + */ +final class ExtractRecycleBin extends Extract { + + private static final Logger logger = Logger.getLogger(ExtractRecycleBin.class.getName()); + + private static final String RECYCLE_BIN_ARTIFACT_NAME = "TSK_RECYCLE_BIN"; //NON-NLS + + private static final int V1_FILE_NAME_OFFSET = 24; + private static final int V2_FILE_NAME_OFFSET = 28; + + @Messages({ + "ExtractRecycleBin_module_name=Recycle Bin" + }) + ExtractRecycleBin() { + this.moduleName = Bundle.ExtractRecycleBin_module_name(); + } + + @Override + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { + // At this time it was decided that we would not include TSK_RECYCLE_BIN + // in the default list of BlackboardArtifact types. + try { + createRecycleBinArtifactType(); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("%s may not have been created.", RECYCLE_BIN_ARTIFACT_NAME), ex); + } + + BlackboardArtifact.Type recycleBinArtifactType; + + try { + recycleBinArtifactType = tskCase.getArtifactType(RECYCLE_BIN_ARTIFACT_NAME); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Unable to retrive custom artifact type %s", RECYCLE_BIN_ARTIFACT_NAME), ex); // NON-NLS + // If this doesn't work bail. + return; + } + + // map SIDs to user names so that we can include that in the artifact + Map userNameMap; + try { + userNameMap = makeUserNameMap(dataSource); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Unable to create OS Account user name map", ex); + // This is not the end of the world we will just continue without + // user names + userNameMap = new HashMap<>(); + } + + FileManager fileManager = Case.getCurrentCase().getServices().getFileManager(); + + // Collect all of the $R files so that we can later easily map them to corresponding $I file + Map> rFileMap; + try { + rFileMap = makeRFileMap(dataSource); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Unable to create $R file map for dataSource: %s", dataSource.getName()), ex); + return; // No $R files, no need to continue; + } + + // Get the $I files + List iFiles; + try { + iFiles = fileManager.findFiles(dataSource, "$I%"); //NON-NLS + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Unable to find recycle bin I files.", ex); //NON-NLS + return; // No need to continue + } + + String tempRARecycleBinPath = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), "recyclebin"); //NON-NLS + + // cycle through the $I files and process each. + for (AbstractFile iFile : iFiles) { + + if (context.dataSourceIngestIsCancelled()) { + return; + } + + processIFile(context, recycleBinArtifactType, iFile, userNameMap, rFileMap, tempRARecycleBinPath); + } + + (new File(tempRARecycleBinPath)).delete(); + } + + /** + * Process each individual iFile. + * + * @param context + * @param recycleBinArtifactType Module created artifact type + * @param iFile The AbstractFile to process + * @param userNameMap Map of user ids to names + * @param tempRARecycleBinPath Temp directory path + */ + private void processIFile(IngestJobContext context, BlackboardArtifact.Type recycleBinArtifactType, AbstractFile iFile, Map userNameMap, Map> rFileMap, String tempRARecycleBinPath) { + String tempFilePath = tempRARecycleBinPath + File.separator + Instant.now().getMillis() + iFile.getName(); + try { + try { + ContentUtils.writeToFile(iFile, new File(tempFilePath)); + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Unable to write %s to temp directory. File name: %s", iFile.getName(), tempFilePath), ex); //NON-NLS + // if we cannot make a copy of the $I file for later processing + // move onto the next file + return; + } + + // get the original name, dates, etc. from the $I file + RecycledFileMetaData metaData; + try { + metaData = parseIFile(tempFilePath); + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Unable to parse iFile %s", iFile.getName()), ex); //NON-NLS + // Unable to parse the $I file move onto the next file + return; + } + + // each user has its own Recyle Bin folder. Figure out the user name based on its name . + String userID = getUserIDFromPath(iFile.getParentPath()); + String userName = ""; + if (!userID.isEmpty()) { + userName = userNameMap.get(userID); + } else { + // If the iFile doesn't have a user ID in its parent + // directory structure then it is not from the recyle bin + return; + } + + // get the corresponding $R file, which is in the same folder and has the file content + String rFileName = iFile.getName().replace("$I", "$R"); //NON-NLS + List rFiles = rFileMap.get(rFileName); + if (rFiles == null) { + return; + } + SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); + for (AbstractFile rFile : rFiles) { + if (context.dataSourceIngestIsCancelled()) { + return; + } + + if (iFile.getParentPath().equals(rFile.getParentPath()) + && iFile.getMetaFlagsAsString().equals(rFile.getMetaFlagsAsString())) { + try { + postArtifact(createArtifact(rFile, recycleBinArtifactType, metaData.getFullWindowsPath(), userName, metaData.getDeletedTimeStamp())); + + // If we are processing a disk image, we will also make a deleted file entry so that the user + // sees the deleted file in its original folder. We re-use the metadata address so that the user + // can see the content. + if (rFile instanceof FsContent) { + // if the user deleted a folder, then we need to recusively go into it. Note the contents of the $R folder + // do not have corresponding $I files anymore. Only the $R folder does. + if (rFile.isDir()) { + AbstractFile directory = getOrMakeFolder(Case.getCurrentCase().getSleuthkitCase(), (FsContent) rFile, metaData.getFullWindowsPath()); + popuplateDeletedDirectory(Case.getCurrentCase().getSleuthkitCase(), directory, rFile.getChildren(), metaData.getFullWindowsPath(), metaData.getDeletedTimeStamp()); + + } else { + AbstractFile folder = getOrMakeFolder(Case.getCurrentCase().getSleuthkitCase(), (FsContent) rFile.getParent(), Paths.get(metaData.getFullWindowsPath()).getParent().toString()); + addFileSystemFile(skCase, (FsContent)rFile, folder, Paths.get(metaData.getFullWindowsPath()).getFileName().toString(), metaData.getDeletedTimeStamp()); + } + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Unable to add attributes to artifact %s", rFile.getName()), ex); //NON-NLS + } + } + } + } finally { + (new File(tempFilePath)).delete(); + } + } + + /** + * Add the children of recycled $R folder to the folder. + * + * @param skCase The current Sleuthkit case + * @param parentFolder The folder to folder the deleted files are to be + * added. + * @param children The recycled children of the $R folder + * @param parentPath String path to the directory the children were + * deleted from + * @param deletedTimeStamp The time at which the files were deleted, + * inherited from the $R file. + * + * @throws TskCoreException + */ + private void popuplateDeletedDirectory(SleuthkitCase skCase, AbstractFile parentFolder, List recycledChildren, String parentPath, long deletedTimeStamp) throws TskCoreException { + if (recycledChildren == null) { + return; + } + + for (Content child : recycledChildren) { + if (child instanceof FsContent) { + FsContent fsContent = (FsContent) child; + if (fsContent.isFile()) { + addFileSystemFile(skCase, fsContent, parentFolder, fsContent.getName(), deletedTimeStamp); + } else if (fsContent.isDir()) { + String newPath = parentPath + "\\" + fsContent.getName(); + AbstractFile childFolder = getOrMakeFolder(skCase, fsContent, parentPath); + popuplateDeletedDirectory(skCase, childFolder, fsContent.getChildren(), newPath, deletedTimeStamp); + } + } + } + } + + /** + * Parse the $I file. + * + * File format prior to Windows 10: + * + * + * + * + * + * + *
OffsetSizeDescription
08Header
88File Size
168Deleted Timestamp
24520File Name
+ * + * File format Windows 10+ + * + * + * + * + * + * + * + *
OffsetSizeDescription
08Header
88File Size
168Deleted TimeStamp
244File Name Length
28varFile Name
+ * + * For versions of Windows prior to 10, header = 0x01. Windows 10+ header == + * 0x02 + * + * @param iFilePath Path to local copy of file in temp folder + * + * @throws FileNotFoundException + * @throws IOException + */ + private RecycledFileMetaData parseIFile(String iFilePath) throws FileNotFoundException, IOException { + byte[] allBytes = Files.readAllBytes(Paths.get(iFilePath)); + + ByteBuffer byteBuffer = ByteBuffer.wrap(allBytes); + byteBuffer.order(ByteOrder.LITTLE_ENDIAN); + + long version = byteBuffer.getLong(); + long fileSize = byteBuffer.getLong(); + long timestamp = byteBuffer.getLong(); + + // Convert from windows FILETIME to Unix Epoch seconds + timestamp = Util.filetimeToMillis(timestamp) / 1000; + + byte[] stringBytes; + + if (version == 1) { + stringBytes = Arrays.copyOfRange(allBytes, V1_FILE_NAME_OFFSET, allBytes.length); + } else { + int fileNameLength = byteBuffer.getInt() * 2; //Twice the bytes for unicode + stringBytes = Arrays.copyOfRange(allBytes, V2_FILE_NAME_OFFSET, V2_FILE_NAME_OFFSET + fileNameLength); + } + + String fileName = new String(stringBytes, "UTF-16LE"); //NON-NLS + + return new RecycledFileMetaData(fileSize, timestamp, fileName); + } + + /** + * Create a map of userids to usernames from the OS Accounts. + * + * @param dataSource + * + * @return A Map of userIDs and userNames + * + * @throws TskCoreException + */ + private Map makeUserNameMap(Content dataSource) throws TskCoreException { + Map userNameMap = new HashMap<>(); + + List accounts = blackboard.getArtifacts(TSK_OS_ACCOUNT.getTypeID(), dataSource.getId()); + + for (BlackboardArtifact account : accounts) { + BlackboardAttribute nameAttribute = getAttributeForArtifact(account, TSK_USER_NAME); + BlackboardAttribute idAttribute = getAttributeForArtifact(account, TSK_USER_ID); + + String userName = nameAttribute != null ? nameAttribute.getDisplayString() : ""; + String userID = idAttribute != null ? idAttribute.getDisplayString() : ""; + + if (!userID.isEmpty()) { + userNameMap.put(userID, userName); + } + } + + return userNameMap; + } + + /** + * Get a list of files that start with $R and create a map of the file to + * their name. + * + * @param dataSource + * + * @return File map + * + * @throws TskCoreException + */ + private Map> makeRFileMap(Content dataSource) throws TskCoreException { + FileManager fileManager = Case.getCurrentCase().getServices().getFileManager(); + List rFiles = fileManager.findFiles(dataSource, "$R%"); + Map> fileMap = new HashMap<>(); + + for (AbstractFile rFile : rFiles) { + String fileName = rFile.getName(); + List fileList = fileMap.get(fileName); + + if (fileList == null) { + fileList = new ArrayList<>(); + fileMap.put(fileName, fileList); + } + + fileList.add(rFile); + } + + return fileMap; + } + + /** + * Helper functions to get the user ID from the iFile parent path. User ids + * will be of the form S-. + * + * @param iFileParentPath String parent path of the iFile + * + * @return String user id + */ + private String getUserIDFromPath(String iFileParentPath) { + int index = iFileParentPath.indexOf('-') - 1; + if (index >= 0) { + return (iFileParentPath.substring(index)).replace("/", ""); + } else { + return ""; + } + } + + /** + * Gets the attribute for the given type from the given artifact. + * + * @param artifact BlackboardArtifact to get the attribute from + * @param type The BlackboardAttribute Type to get + * + * @return BlackboardAttribute for given artifact and type + * + * @throws TskCoreException + */ + private BlackboardAttribute getAttributeForArtifact(BlackboardArtifact artifact, BlackboardAttribute.ATTRIBUTE_TYPE type) throws TskCoreException { + return artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.fromID(type.getTypeID()))); + } + + /** + * Create TSK_RECYCLE_BIN artifact type. + * + * @throws TskCoreException + */ + private void createRecycleBinArtifactType() throws TskCoreException { + try { + tskCase.addBlackboardArtifactType(RECYCLE_BIN_ARTIFACT_NAME, "Recycle Bin"); //NON-NLS + } catch (TskDataException ex) { + logger.log(Level.INFO, String.format("%s may have already been defined for this case", RECYCLE_BIN_ARTIFACT_NAME)); + } + + } + + /** + * Create the new artifact for the give rFile + * + * @param rFile AbstractFile to create the artifact for + * @param type Type of artifact to create + * @param fileName The original path of the deleted file + * @param userName The name of the user that deleted the file + * @param dateTime The time in epoch seconds that the file was deleted + * + * @return Newly created artifact + * + * @throws TskCoreException + */ + private BlackboardArtifact createArtifact(AbstractFile rFile, BlackboardArtifact.Type type, String fileName, String userName, long dateTime) throws TskCoreException { + BlackboardArtifact bba = rFile.newArtifact(type.getTypeID()); + bba.addAttribute(new BlackboardAttribute(TSK_PATH, getName(), fileName)); + bba.addAttribute(new BlackboardAttribute(TSK_DATETIME_DELETED, getName(), dateTime)); + bba.addAttribute(new BlackboardAttribute(TSK_USER_NAME, getName(), userName == null || userName.isEmpty() ? "" : userName)); + return bba; + } + + /** + * Returns a folder for the given path. If the path does not exist the + * the folder is created. Recursively makes as many parent folders as needed. + * + * @param skCase + * @param dataSource + * @param path + * + * @return AbstractFile for the given path. + * + * @throws TskCoreException + */ + private AbstractFile getOrMakeFolder(SleuthkitCase skCase, FsContent dataSource, String path) throws TskCoreException { + + String parentPath = getParentPath(path); + String folderName = getFileName(path); + + List files = null; + if (parentPath != null) { + if (!parentPath.equals("/")) { + parentPath = parentPath + "/"; + } + + files = skCase.findAllFilesWhere(String.format("fs_obj_id=%s AND parent_path='%s' AND name='%s'", + dataSource.getFileSystemId(), SleuthkitCase.escapeSingleQuotes(parentPath), folderName != null ? SleuthkitCase.escapeSingleQuotes(folderName) : "")); + } else { + files = skCase.findAllFilesWhere(String.format("fs_obj_id=%s AND parent_path='/' AND name=''", dataSource.getFileSystemId())); + } + + if (files == null || files.isEmpty()) { + AbstractFile parent = getOrMakeFolder(skCase, dataSource, parentPath); + return skCase.addVirtualDirectory(parent.getId(), folderName); + } else { + return files.get(0); + } + } + + /** + * Adds a new file system file that is unallocated and maps to the original + * file in recycle bin directory. + * + * @param skCase The current case. + * @param recycleBinFile The file from the recycle bin. + * @param parentDir The directory that the recycled file was deleted. + * @param fileName The name of the file. + * @param deletedTime The time the file was deleted. + * + * @throws TskCoreException + */ + private void addFileSystemFile(SleuthkitCase skCase, FsContent recycleBinFile, Content parentDir, String fileName, long deletedTime) throws TskCoreException { + skCase.addFileSystemFile( + recycleBinFile.getDataSourceObjectId(), + recycleBinFile.getFileSystemId(), + fileName, + recycleBinFile.getMetaAddr(), + (int) recycleBinFile.getMetaSeq(), + recycleBinFile.getAttrType(), + recycleBinFile.getAttributeId(), + TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC, + (short) (TskData.TSK_FS_META_FLAG_ENUM.UNALLOC.getValue() | TskData.TSK_FS_META_FLAG_ENUM.USED.getValue()), + recycleBinFile.getSize(), + recycleBinFile.getCtime(), recycleBinFile.getCrtime(), recycleBinFile.getAtime(), deletedTime, + true, parentDir); + } + + /** + * Clean up the windows path string to match what the autopsy db uses. + * + * @param path The file\folder path to normalize + * + * @return New path string with the root removed (ie X:) and the slashes + * changed from windows to unix. + */ + String normalizeFilePath(String pathString) { + if (pathString == null || pathString.isEmpty()) { + return null; + } + + Path path = Paths.get(pathString); + int nameCount = path.getNameCount(); + if(nameCount > 0) { + String rootless = "/" + path.subpath(0, nameCount); + return rootless.replace("\\", "/"); + } else { + return "/"; + } + } + + /** + * Helper function get from the given path either the file name or + * the last directory in the path. + * + * @param filePath The file\directory path + * + * @return If file path, returns the file name. If directory path the + * The last directory in the path is returned. + */ + String getFileName(String filePath) { + Path fileNamePath = Paths.get(filePath).getFileName(); + if (fileNamePath != null) { + return fileNamePath.toString(); + } + return filePath; + } + + /** + * Returns the parent path for the given path. + * + * @param path Path string + * + * @return The parent path for the given path. + */ + String getParentPath(String path) { + Path parentPath = Paths.get(path).getParent(); + if (parentPath != null) { + return normalizeFilePath(parentPath.toString()); + } + return null; + } + + /** + * Stores the data from the $I files. + */ + final class RecycledFileMetaData { + + private final long fileSize; + private final long deletedTimeStamp; + private final String fileName; + + /** + * Constructs a new instance. + * + * @param fileSize Size of the deleted file. + * @param deletedTimeStamp Time the file was deleted. + * @param fileName Name of the deleted file. + */ + RecycledFileMetaData(Long fileSize, long deletedTimeStamp, String fileName) { + this.fileSize = fileSize; + this.deletedTimeStamp = deletedTimeStamp; + this.fileName = fileName; + } + + /** + * Returns the size of the recycled file. + * + * @return Size of deleted file + */ + long getFileSize() { + return fileSize; + } + + /** + * Returns the time the file was deleted. + * + * @return deleted time in epoch seconds. + */ + long getDeletedTimeStamp() { + return deletedTimeStamp; + } + + /** + * Returns the full path to the deleted file or folder. This path will + * include the drive letter, ie C:\ + * + * @return String name of the deleted file + */ + String getFullWindowsPath() { + return fileName.trim(); + } + } +} diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 6e4d0fc4c7..aaaac4415f 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -68,13 +68,19 @@ import org.openide.util.Lookup; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; +import org.sleuthkit.autopsy.recentactivity.ShellBagParser.ShellBag; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_MODIFIED; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException; import org.sleuthkit.datamodel.Report; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskDataException; /** * Extract windows registry data using regripper. Runs two versions of @@ -85,7 +91,10 @@ import org.sleuthkit.datamodel.TskCoreException; @NbBundle.Messages({ "RegRipperNotFound=Autopsy RegRipper executable not found.", "RegRipperFullNotFound=Full version RegRipper executable not found.", - "Progress_Message_Analyze_Registry=Analyzing Registry Files" + "Progress_Message_Analyze_Registry=Analyzing Registry Files", + "Shellbag_Artifact_Display_Name=Shell Bags", + "Shellbag_Key_Attribute_Display_Name=Key", + "Shellbag_Last_Write_Attribute_Display_Name=Last Write" }) class ExtractRegistry extends Extract { @@ -132,6 +141,14 @@ class ExtractRegistry extends Extract { private final Path rrFullHome; // Path to the full version of RegRipper private Content dataSource; private IngestJobContext context; + + private static final String SHELLBAG_ARTIFACT_NAME = "RA_SHELL_BAG"; //NON-NLS + private static final String SHELLBAG_ATTRIBUTE_LAST_WRITE = "RA_SHELL_BAG_LAST_WRITE"; //NON-NLS + private static final String SHELLBAG_ATTRIBUTE_KEY= "RA_SHELL_BAG_KEY"; //NON-NLS + + BlackboardArtifact.Type shellBagArtifactType = null; + BlackboardAttribute.Type shellBagKeyAttributeType = null; + BlackboardAttribute.Type shellBagLastWriteAttributeType = null; ExtractRegistry() throws IngestModuleException { moduleName = NbBundle.getMessage(ExtractIE.class, "ExtractRegistry.moduleName.text"); @@ -195,6 +212,13 @@ class ExtractRegistry extends Extract { } catch (TskCoreException ex) { logger.log(Level.WARNING, "Error fetching 'ntuser.dat' file."); //NON-NLS } + + // find the user-specific ntuser-dat files + try { + allRegistryFiles.addAll(fileManager.findFiles(dataSource, "usrclass.dat")); //NON-NLS + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Error finding 'usrclass.dat' files."), ex); //NON-NLS + } // find the system hives' String[] regFileNames = new String[]{"system", "software", "security", "sam"}; //NON-NLS @@ -204,7 +228,7 @@ class ExtractRegistry extends Extract { } catch (TskCoreException ex) { String msg = NbBundle.getMessage(this.getClass(), "ExtractRegistry.findRegFiles.errMsg.errReadingFile", regFileName); - logger.log(Level.WARNING, msg); + logger.log(Level.WARNING, msg, ex); this.addErrorMessage(this.getName() + ": " + msg); } } @@ -282,6 +306,13 @@ class ExtractRegistry extends Extract { this.addErrorMessage( NbBundle.getMessage(this.getClass(), "ExtractRegistry.analyzeRegFiles.failedParsingResults", this.getName(), regFileName)); + } else if (regFileNameLocal.toLowerCase().contains("ntuser") || regFileNameLocal.toLowerCase().contains("usrclass")) { + try { + List shellbags = ShellBagParser.parseShellbagOutput(regOutputFiles.fullPlugins); + createShellBagArtifacts(regFile, shellbags); + } catch (IOException | TskCoreException ex) { + logger.log(Level.WARNING, String.format("Unable to get shell bags from file %s", regOutputFiles.fullPlugins), ex); + } } try { Report report = currentCase.addReport(regOutputFiles.fullPlugins, @@ -340,6 +371,8 @@ class ExtractRegistry extends Extract { fullType = "sam"; //NON-NLS } else if (regFilePath.toLowerCase().contains("security")) { //NON-NLS fullType = "security"; //NON-NLS + }else if (regFilePath.toLowerCase().contains("usrclass")) { //NON-NLS + fullType = "usrclass"; //NON-NLS } else { return regOutputFiles; } @@ -398,6 +431,7 @@ class ExtractRegistry extends Extract { */ private boolean parseAutopsyPluginOutput(String regFilePath, AbstractFile regFile) { FileInputStream fstream = null; + List newArtifacts = new ArrayList<>(); try { // Read the file in and create a Document and elements File regfile = new File(regFilePath); @@ -414,7 +448,7 @@ class ExtractRegistry extends Extract { String stringdoc = startdoc + result + enddoc; DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); Document doc = builder.parse(new InputSource(new StringReader(stringdoc))); - + // cycle through the elements in the doc Element oroot = doc.getDocumentElement(); NodeList children = oroot.getChildNodes(); @@ -505,7 +539,7 @@ class ExtractRegistry extends Extract { String Tempdate = installtime.toString(); installtime = Long.valueOf(Tempdate) / MS_IN_SEC; } catch (ParseException e) { - logger.log(Level.SEVERE, "RegRipper::Conversion on DateTime -> ", e); //NON-NLS + logger.log(Level.WARNING, "RegRipper::Conversion on DateTime -> ", e); //NON-NLS } } break; @@ -531,8 +565,7 @@ class ExtractRegistry extends Extract { BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_OS_INFO); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } else { results.get(0).addAttributes(bbattributes); } @@ -582,8 +615,7 @@ class ExtractRegistry extends Extract { BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_OS_INFO); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } else { results.get(0).addAttributes(bbattributes); } @@ -621,13 +653,12 @@ class ExtractRegistry extends Extract { BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_OS_INFO); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } else { results.get(0).addAttributes(bbattributes); } } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error adding os info artifact to blackboard."); //NON-NLS + logger.log(Level.SEVERE, "Error adding os info artifact to blackboard.", ex); //NON-NLS } break; default: @@ -673,8 +704,7 @@ class ExtractRegistry extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_ID, parentModuleName, value)); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding device attached artifact to blackboard.", ex); //NON-NLS } @@ -688,7 +718,7 @@ class ExtractRegistry extends Extract { itemMtime /= MS_IN_SEC; } } catch (ParseException ex) { - logger.log(Level.WARNING, "Failed to parse epoch time for installed program artifact.", ex); //NON-NLS + logger.log(Level.SEVERE, "Failed to parse epoch time for installed program artifact.", ex); //NON-NLS } try { @@ -697,8 +727,7 @@ class ExtractRegistry extends Extract { BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_INSTALLED_PROG); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding installed program artifact to blackboard.", ex); //NON-NLS } @@ -717,8 +746,7 @@ class ExtractRegistry extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, parentModuleName, artnode.getNodeName())); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding recent object artifact to blackboard.", ex); //NON-NLS } @@ -758,7 +786,7 @@ class ExtractRegistry extends Extract { } } } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Error getting existing os account artifact", ex); + logger.log(Level.SEVERE, "Error getting existing os account artifact", ex); } if (bbart == null) { //create new artifact @@ -784,8 +812,7 @@ class ExtractRegistry extends Extract { } } bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding account artifact to blackboard.", ex); //NON-NLS } @@ -801,8 +828,7 @@ class ExtractRegistry extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REMOTE_PATH, parentModuleName, remoteName)); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding network artifact to blackboard.", ex); //NON-NLS } @@ -817,8 +843,7 @@ class ExtractRegistry extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_ID, parentModuleName, adapter)); BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_WIFI_NETWORK); bbart.addAttributes(bbattributes); - // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding SSID artifact to blackboard.", ex); //NON-NLS } @@ -830,7 +855,7 @@ class ExtractRegistry extends Extract { break; default: - logger.log(Level.WARNING, "Unrecognized node name: {0}", dataType); //NON-NLS + logger.log(Level.SEVERE, "Unrecognized node name: {0}", dataType); //NON-NLS break; } } @@ -840,13 +865,13 @@ class ExtractRegistry extends Extract { } // for return true; } catch (FileNotFoundException ex) { - logger.log(Level.SEVERE, "Error finding the registry file.", ex); //NON-NLS + logger.log(Level.WARNING, String.format("Error finding the registry file: %s", regFilePath), ex); //NON-NLS } catch (SAXException ex) { - logger.log(Level.SEVERE, "Error parsing the registry XML.", ex); //NON-NLS + logger.log(Level.WARNING, String.format("Error parsing the registry XML: %s", regFilePath), ex); //NON-NLS } catch (IOException ex) { - logger.log(Level.SEVERE, "Error building the document parser.", ex); //NON-NLS + logger.log(Level.WARNING, String.format("Error building the document parser: %s", regFilePath), ex); //NON-NLS } catch (ParserConfigurationException ex) { - logger.log(Level.SEVERE, "Error configuring the registry parser.", ex); //NON-NLS + logger.log(Level.WARNING, String.format("Error configuring the registry parser: %s", regFilePath), ex); //NON-NLS } finally { try { if (fstream != null) { @@ -854,6 +879,8 @@ class ExtractRegistry extends Extract { } } catch (IOException ex) { } + + postArtifacts(newArtifacts); } return false; } @@ -869,6 +896,7 @@ class ExtractRegistry extends Extract { */ private boolean parseSamPluginOutput(String regFilePath, AbstractFile regAbstractFile) { File regfile = new File(regFilePath); + List newArtifacts = new ArrayList<>(); try (BufferedReader bufferedReader = new BufferedReader(new FileReader(regfile))) { // Read the file in and create a Document and elements String userInfoSection = "User Information"; @@ -909,23 +937,25 @@ class ExtractRegistry extends Extract { } } } + //add remaining userinfos as accounts; for (Map userInfo : userInfoMap.values()) { BlackboardArtifact bbart = regAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT); bbart.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userInfo.get(SID_KEY)), false)); // index the artifact for keyword search - postArtifact(bbart); + newArtifacts.add(bbart); } - //store set of attributes to make artifact for later in collection of artifact like objects return true; } catch (FileNotFoundException ex) { - logger.log(Level.SEVERE, "Error finding the registry file.", ex); //NON-NLS + logger.log(Level.WARNING, "Error finding the registry file.", ex); //NON-NLS } catch (IOException ex) { - logger.log(Level.SEVERE, "Error building the document parser: {0}", ex); //NON-NLS + logger.log(Level.WARNING, "Error building the document parser: {0}", ex); //NON-NLS } catch (ParseException ex) { - logger.log(Level.SEVERE, "Error parsing the the date from the registry file", ex); //NON-NLS + logger.log(Level.WARNING, "Error parsing the the date from the registry file", ex); //NON-NLS } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error updating TSK_OS_ACCOUNT artifacts to include newly parsed data.", ex); //NON-NLS + logger.log(Level.WARNING, "Error updating TSK_OS_ACCOUNT artifacts to include newly parsed data.", ex); //NON-NLS + } finally { + postArtifacts(newArtifacts); } return false; } @@ -1108,7 +1138,9 @@ class ExtractRegistry extends Extract { line = bufferedReader.readLine(); while (line != null && !line.isEmpty()) { entry = getSAMKeyValue(line); - userInfo.put(entry.getKey(), entry.getValue()); + if (entry != null) { + userInfo.put(entry.getKey(), entry.getValue()); + } line = bufferedReader.readLine(); } users.add(userInfo); @@ -1119,7 +1151,126 @@ class ExtractRegistry extends Extract { } } + /** + * Create the shellbag artifacts from the list of ShellBag objects. + * + * @param regFile The data source file + * @param shellbags List of shellbags from source file + * + * @throws TskCoreException + */ + void createShellBagArtifacts(AbstractFile regFile, List shellbags) throws TskCoreException { + List artifacts = new ArrayList<>(); + try{ + for (ShellBag bag : shellbags) { + Collection attributes = new ArrayList<>(); + BlackboardArtifact artifact = regFile.newArtifact(getShellBagArtifact().getTypeID()); + attributes.add(new BlackboardAttribute(TSK_PATH, getName(), bag.getResource())); + attributes.add(new BlackboardAttribute(getKeyAttribute(), getName(), bag.getKey())); + + long time; + time = bag.getLastWrite(); + if (time != 0) { + attributes.add(new BlackboardAttribute(getLastWriteAttribute(), getName(), time)); + } + + time = bag.getModified(); + if (time != 0) { + attributes.add(new BlackboardAttribute(TSK_DATETIME_MODIFIED, getName(), time)); + } + + time = bag.getCreated(); + if (time != 0) { + attributes.add(new BlackboardAttribute(TSK_DATETIME_CREATED, getName(), time)); + } + + time = bag.getAccessed(); + if (time != 0) { + attributes.add(new BlackboardAttribute(TSK_DATETIME_ACCESSED, getName(), time)); + } + + artifact.addAttributes(attributes); + + artifacts.add(artifact); + } + } finally { + postArtifacts(artifacts); + } + } + + /** + * Returns the custom Shellbag artifact type or creates it if it does not + * currently exist. + * + * @return BlackboardArtifact.Type for shellbag artifacts + * + * @throws TskCoreException + */ + private BlackboardArtifact.Type getShellBagArtifact() throws TskCoreException { + if (shellBagArtifactType == null) { + shellBagArtifactType = tskCase.getArtifactType(SHELLBAG_ARTIFACT_NAME); + + if(shellBagArtifactType == null) { + try { + tskCase.addBlackboardArtifactType(SHELLBAG_ARTIFACT_NAME, Bundle.Shellbag_Artifact_Display_Name()); //NON-NLS + } catch (TskDataException ex) { + // Artifact already exists + logger.log(Level.INFO, String.format("%s may have already been defined for this case", SHELLBAG_ARTIFACT_NAME)); + } + + shellBagArtifactType = tskCase.getArtifactType(SHELLBAG_ARTIFACT_NAME); + } + } + + return shellBagArtifactType; + } + + /** + * Gets the custom BlackboardAttribute type. The attribute type is created + * if it does not currently exist. + * + * @return The BlackboardAttribute type + * + * @throws TskCoreException + */ + private BlackboardAttribute.Type getLastWriteAttribute() throws TskCoreException { + if (shellBagLastWriteAttributeType == null) { + try { + shellBagLastWriteAttributeType = tskCase.addArtifactAttributeType(SHELLBAG_ATTRIBUTE_LAST_WRITE, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DATETIME, + Bundle.Shellbag_Last_Write_Attribute_Display_Name()); + } catch (TskDataException ex) { + // Attribute already exists get it from the case + shellBagLastWriteAttributeType = tskCase.getAttributeType(SHELLBAG_ATTRIBUTE_LAST_WRITE); + } + } + return shellBagLastWriteAttributeType; + } + + /** + * Gets the custom BlackboardAttribute type. The attribute type is created + * if it does not currently exist. + * + * @return The BlackboardAttribute type + * + * @throws TskCoreException + */ + private BlackboardAttribute.Type getKeyAttribute() throws TskCoreException { + if (shellBagKeyAttributeType == null) { + try { + shellBagKeyAttributeType = tskCase.addArtifactAttributeType(SHELLBAG_ATTRIBUTE_KEY, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + Bundle.Shellbag_Key_Attribute_Display_Name()); + } catch (TskDataException ex) { + // The attribute already exists get it from the case + shellBagKeyAttributeType = tskCase.getAttributeType(SHELLBAG_ATTRIBUTE_KEY); + } + } + return shellBagKeyAttributeType; + } + + /** * Maps the user groups to the sid that are a part of them. * * @param bufferedReader diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java index db815e9274..444a6d638d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java @@ -77,6 +77,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule { Extract dataSourceAnalyzer = new DataSourceUsageAnalyzer(); Extract safari = new ExtractSafari(); Extract zoneInfo = new ExtractZoneIdentifier(); + Extract recycleBin = new ExtractRecycleBin(); extractors.add(chrome); extractors.add(firefox); @@ -89,6 +90,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule { extractors.add(osExtract); // this needs to run before the DataSourceUsageAnalyzer extractors.add(dataSourceAnalyzer); //this needs to run after ExtractRegistry and ExtractOs extractors.add(zoneInfo); // this needs to run after the web browser modules + extractors.add(recycleBin); // this needs to run after ExtractRegistry and ExtractOS browserExtractors.add(chrome); browserExtractors.add(firefox); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ShellBagParser.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ShellBagParser.java new file mode 100755 index 0000000000..5f8f2eb38f --- /dev/null +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ShellBagParser.java @@ -0,0 +1,362 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * + * Copyright 2012 42six Solutions. + * Contact: aebadirad 42six com + * Project Contact/Architect: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.recentactivity; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; + +/** + * Parse the ntuser and ursclass regripper output files for shellbags. + */ +class ShellBagParser { + private static final Logger logger = Logger.getLogger(ShellBagParser.class.getName()); + + private static final SimpleDateFormat DATE_TIME_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault()); + // Last Write date\time format from itempos plugin + private static final SimpleDateFormat DATE_TIME_FORMATTER2 = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyyy", Locale.getDefault()); + + private ShellBagParser() { + } + + /** + * Parse the given file for shell bags. + * + * @param regFilePath Regripper output file + * + * @return List of the found shellbags + * + * @throws FileNotFoundException + * @throws IOException + */ + static List parseShellbagOutput(String regFilePath) throws FileNotFoundException, IOException { + List shellbags = new ArrayList<>(); + File regfile = new File(regFilePath); + + ShellBagParser sbparser = new ShellBagParser(); + + try (BufferedReader reader = new BufferedReader(new FileReader(regfile))) { + String line = reader.readLine(); + while (line != null) { + line = line.trim(); + + if (line.matches("^shellbags_xp v.*")) { + shellbags.addAll(sbparser.parseShellBagsXP(reader)); + } else if (line.matches("^shellbags v.*")) { + shellbags.addAll(sbparser.parseShellBags(reader)); + } else if (line.matches("^itempos.*")) { + shellbags.addAll(sbparser.parseItempos(reader)); + } + + line = reader.readLine(); + } + } + + return shellbags; + } + + /** + * Parse the output from the shellbag_xp plugin. + * + * @param reader File reader + * + * @return List of found shellbags + * + * @throws IOException + */ + List parseShellBagsXP(BufferedReader reader) throws IOException { + List shellbags = new ArrayList<>(); + String line = reader.readLine(); + + while (line != null && !isSectionSeparator(line)) { + + if (isShellbagXPDataLine(line)) { + String[] tokens = line.split("\\|"); + if (tokens.length >= 6) { + shellbags.add(new ShellBag(tokens[5].trim(), "Software\\Microsoft\\Windows\\ShellNoRoam\\BagMRU", tokens[0].trim(), tokens[1].trim(), tokens[2].trim(), tokens[3].trim())); + } + } + + line = reader.readLine(); + } + + return shellbags; + } + + /** + * Parse the output of the shellbags regripper plugin. + * + * @param reader + * @return List of found shellbags + * + * @throws IOException + */ + List parseShellBags(BufferedReader reader) throws IOException { + List shellbags = new ArrayList<>(); + String line = reader.readLine(); + String regPath = "Local Settings\\Software\\Microsoft\\Windows\\Shell\\BagMRU"; + + while (line != null && !isSectionSeparator(line)) { + + if (isShellbagDataLine(line)) { + String[] tokens = line.split("\\|"); + String path = tokens[6].replaceAll("\\[.*?\\]", "").trim(); + int index = line.lastIndexOf('['); + String endstuff = ""; + if (index != -1) { + endstuff = line.substring(index, line.length() - 1).replace("[Desktop", ""); + } + if (tokens.length >= 7) { + shellbags.add(new ShellBag(path, regPath + endstuff, tokens[0].trim(), tokens[1].trim(), tokens[2].trim(), tokens[3].trim())); + } + } + + line = reader.readLine(); + } + + return shellbags; + } + + /** + * Parse the output of the Itempos regripper plugin. + * + * @param reader + * + * @return List of found shell bags. + * + * @throws IOException + */ + List parseItempos(BufferedReader reader) throws IOException { + List shellbags = new ArrayList<>(); + String bagpath = ""; + String lastWrite = ""; + String line = reader.readLine(); + + while (line != null && !isSectionSeparator(line)) { + + if (isItemposDataLine(line)) { + String[] tokens = line.split("\\|"); + if (tokens.length >= 5) { + shellbags.add(new ShellBag(tokens[4].trim(), bagpath, lastWrite, tokens[1].trim(), tokens[2].trim(), tokens[3].trim())); + } + } else if (line.contains("Software\\")) { + bagpath = line.trim(); + lastWrite = ""; + } else if (line.contains("LastWrite:")) { + lastWrite = line.replace("LastWrite:", "").trim(); + } + + line = reader.readLine(); + } + + return shellbags; + } + + /** + * Return whether or not the given line is a plugin output separator. + * + * The format of the plugin output separators is: + * ---------------------------------------- + * + * @param line + * + * @return True if the line is a section separator + */ + boolean isSectionSeparator(String line) { + if (line == null || line.isEmpty()) { + return false; + } + + return line.trim().matches("^-+"); + } + + /** + * This data rows from the itempos plugin are in the format: + * | | | | + * The times are in the format YYYY-MM-dd HH:mm:ss + * + * @param line + * + * @return + */ + boolean isItemposDataLine(String line) { + return line.matches("^\\d*?\\s*?\\|.*?\\|.*?\\|.*?\\|.*?"); + } + + /** + * The data rows from the shellbags_xp plug look like + * | | | | + * | + * + * The times are in the format YYYY-MM-dd HH:mm:ss + * + * @param line + * + * @return + */ + boolean isShellbagXPDataLine(String line) { + return line.matches("^(\\d+?.*?\\s*? | \\s*?)\\|.*?\\|.*?\\|.*?\\|.*?\\|.*?"); + } + + /** + * The data rows from the shellbags plug look like + * | | | | + * | + * + * The times are in the format YYYY-MM-dd HH:mm:ss + * + * @param line + * + * @return + */ + boolean isShellbagDataLine(String line) { + return line.matches("^(\\d+?.*?\\s*? | \\s*?)\\|.*?\\|.*?\\|.*?\\|.*?\\|.*?\\|.*?"); + } + + /** + * Class to hold the shell bag data. + * + */ + class ShellBag { + + private final String resource; + private final String key; + private final String lastWrite; + private final String modified; + private final String accessed; + private final String created; + + /** + * Creates a new shell bag object. + * + * Any of the parameters can be ""; + * + * @param resource String from the "Resource" or "Name" column, depending on the plug in + * @param key String registry key value + * @param lastWrite Depending on the plug in lastWrite is either Last write value or the MRU Time value + * @param modified Modified time string + * @param accessed Accessed time string + * @param created Created time string + */ + ShellBag(String resource, String key, String lastWrite, String modified, String accessed, String created) { + this.resource = resource; + this.key = key; + this.lastWrite = lastWrite; + this.accessed = accessed; + this.modified = modified; + this.created = created; + } + + /** + * Returns the resource string. + * + * @return The shellbag resource or empty string. + */ + String getResource() { + return resource == null ? "" : resource; + } + + /** + * Returns the key string. + * + * @return The shellbag key or empty string. + */ + String getKey() { + return key == null ? "" : key; + } + + /** + * Returns the last time in seconds since java epoch or + * 0 if no valid time was found. + * + * @return The time in seconds or 0 if no valid time. + */ + long getLastWrite() { + return parseDateTime(lastWrite); + } + + /** + * Returns the last time in seconds since java epoch or + * 0 if no valid time was found. + * + * @return The time in seconds or 0 if no valid time. + */ + long getModified() { + return parseDateTime(modified); + } + + /** + * Returns the last time in seconds since java epoch or + * 0 if no valid time was found. + * + * @return The time in seconds or 0 if no valid time. + */ + long getAccessed() { + return parseDateTime(accessed); + } + + /** + * Returns the last time in seconds since java epoch or + * 0 if no valid time was found. + * + * @return The time in seconds or 0 if no valid time. + */ + long getCreated() { + return parseDateTime(created); + } + + /** + * Returns the date\time in seconds from epoch for the given string with + * format yyyy-MM-dd HH:mm:ss; + * + * @param dateTimeString String of format yyyy-MM-dd HH:mm:ss + * + * @return time in seconds from java epoch + */ + long parseDateTime(String dateTimeString) { + if (!dateTimeString.isEmpty()) { + try { + return DATE_TIME_FORMATTER.parse(dateTimeString).getTime() / 1000; + } catch (ParseException ex) { + // The parse of the string may fail because there are two possible formats. + } + + try { + return DATE_TIME_FORMATTER2.parse(dateTimeString).getTime() / 1000; + } catch (ParseException ex) { + logger.log(Level.WARNING, String.format("ShellBag parse failure. %s is not formated as expected.", dateTimeString), ex); + } + } + return 0; + } + } + +} diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index a6c6416416..ff95c60ca8 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -51,6 +51,12 @@ import org.sleuthkit.datamodel.TskCoreException; class Util { private static Logger logger = Logger.getLogger(Util.class.getName()); + + /** Difference between Filetime epoch and Unix epoch (in ms). */ + private static final long FILETIME_EPOCH_DIFF = 11644473600000L; + + /** One millisecond expressed in units of 100s of nanoseconds. */ + private static final long FILETIME_ONE_MILLISECOND = 10 * 1000; private Util() { } @@ -176,4 +182,16 @@ class Util { } return results; } + + /** + * Converts a windows FILETIME to java-unix epoch milliseconds + * + * @param filetime 100 nanosecond intervals from jan 1, 1601 + * + * @return java-unix epoch milliseconds + */ + static long filetimeToMillis(final long filetime) { + return (filetime / FILETIME_ONE_MILLISECOND) - FILETIME_EPOCH_DIFF; + } + } diff --git a/Running_Linux_OSX.txt b/Running_Linux_OSX.txt index 1edf48eb13..56ea6a5888 100644 --- a/Running_Linux_OSX.txt +++ b/Running_Linux_OSX.txt @@ -9,13 +9,26 @@ The following need to be done at least once. They do not need to be repeated for -- Linux: % sudo apt-get install testdisk -- OS X: % brew install testdisk -- Install Oracle Java and set JAVA_HOME. --- Linux: Use the instructions here: https://medium.com/coderscorner/installing-oracle-java-8-in-ubuntu-16-10-845507b13343 - NOTE: You may need to log out and back in again after setting JAVA_HOME before the Autopsy - unix_setup.sh script can see the value. --- OS X: Use The Oracle website: https://www.java.com/ - Set JAVA_HOME with something like: export JAVA_HOME=`/usr/libexec/java_home` in .bash_profile +- Install a Java 8 JRE and JavaFX 8 and set JAVA_HOME. +-- Linux: Any Java 8 version of OpenJDK/OpenJFX distribution should suffice. The following instructions use the Zulu Community distribution. + 1. Download a 64 bit Java 8 JRE for your specific platform from https://www.azul.com/downloads/zulu-community + 2. Install the JRE. e.g. % sudo apt install ./zulu8.40.0.25-ca-jre8.0.222-linux_amd64.deb + 3. Download a 64 bit Java 8 JavaFX for your specific platform from the same location. + 4. Extract the contents of the JavaFX archive into the folder where the JRE was installed. + e.g. % sudo tar xzf ~/Downloads/zulu8.40.0.25-ca-fx-jre8.0.222-linux_x64.tar.gz -C /usr/lib/jvm/zre-8-amd64 --strip-components=1 + NOTE: You may need to log out and back in again after setting JAVA_HOME before the Autopsy + unix_setup.sh script can see the value. + +-- OS X: Any Java 8 version of OpenJDK/OpenJFX distribution should suffice. + 1. Install a 64 bit Java 8 JRE. + % brew tap adoptopenjdk/openjdk + % brew cask install adoptopenjdk8 + 2. Download a 64 bit Java 8 JavaFX for macOS from https://www.azul.com/downloads/zulu-community + 3. Extract the contents of the JavaFX archive into the folder where the JRE was installed. + e.g. % sudo tar xf ~/Downloads/zulu8.40.0.25-ca-fx-jre8.0.222-macosx_x64.tar.gz -C /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home--strip-components=1 + 4. Confirm Java 8 is being found by running 'java -version' + 5. Set JAVA_HOME environment variable to location of JRE installation. * Install The Sleuth Kit Java Bindings * diff --git a/TSKVersion.xml b/TSKVersion.xml index 7e65087fae..4bbaf0c0d0 100644 --- a/TSKVersion.xml +++ b/TSKVersion.xml @@ -1,3 +1,3 @@ - + diff --git a/Testing/nbproject/project.xml b/Testing/nbproject/project.xml index 4ed9232a8f..41dc8b253e 100644 --- a/Testing/nbproject/project.xml +++ b/Testing/nbproject/project.xml @@ -47,7 +47,7 @@ 10 - 10.15 + 10.17 diff --git a/appveyor.yml b/appveyor.yml index 03eaf5a4ae..d7cba33584 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -45,8 +45,5 @@ build_script: - ps: popd - cd %APPVEYOR_BUILD_FOLDER% - cmd: ant -q build - - cd Core - - cmd: ant -q test - - cd .. test: off diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index 76d2f2846d..4fd4fb33d8 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 07 Jun 2019 14:47:12 -0400 +#Fri, 04 Oct 2019 14:30:10 -0400 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 @@ -8,4 +8,4 @@ SplashRunningTextBounds=0,289,538,18 SplashRunningTextColor=0x0 SplashRunningTextFontSize=19 -currentVersion=Autopsy 4.11.0 +currentVersion=Autopsy 4.13.0 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index 1da6c75411..17f4cb7436 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,4 +1,4 @@ #Updated by build script -#Fri, 07 Jun 2019 14:47:12 -0400 -CTL_MainWindow_Title=Autopsy 4.11.0 -CTL_MainWindow_Title_No_Project=Autopsy 4.11.0 +#Fri, 04 Oct 2019 14:30:10 -0400 +CTL_MainWindow_Title=Autopsy 4.13.0 +CTL_MainWindow_Title_No_Project=Autopsy 4.13.0 diff --git a/docs/doxygen-user/Doxyfile b/docs/doxygen-user/Doxyfile index 7991a0cd95..1bb3773a70 100644 --- a/docs/doxygen-user/Doxyfile +++ b/docs/doxygen-user/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = "Autopsy User Documentation" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 4.12.0 +PROJECT_NUMBER = 4.13.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -1025,7 +1025,7 @@ GENERATE_HTML = YES # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_OUTPUT = 4.12.0 +HTML_OUTPUT = 4.13.0 # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). diff --git a/docs/doxygen-user/android_analyzer.dox b/docs/doxygen-user/android_analyzer.dox index 8b6b75fa76..a849e089ff 100644 --- a/docs/doxygen-user/android_analyzer.dox +++ b/docs/doxygen-user/android_analyzer.dox @@ -11,14 +11,29 @@ The module should be able to extract the following: - Text messages / SMS / MMS - Call Logs - Contacts -- Tango Messages -- Words with Friends Messages - GPS from the browser and Google Maps - GPS from cache.wifi and cache.cell files +The module may also extract data from the following apps: +- Facebook Messenger +- IMO +- Line +- Opera Browser +- Orux Maps +- S-Browser +- ShareIt +- Skype +- Tango +- TextNow +- Viber +- WhatsApp +- Words with Friends +- Xender +- Zapya + NOTE: These database formats vary by version of OS and different vendors can place the databases in different places. Autopsy may not support all versions and vendors. -NOTE: This module is not exhaustive with its support for Android. It was created as a starting point for others to contribute plug-ins for 3rd party apps. See the Developer docs for information on writing modules. +NOTE: This module is not exhaustive with its support for Android. It was created as a starting point for others to contribute plug-ins for 3rd party apps. See the Developer docs for information on writing modules. Configuration @@ -45,7 +60,6 @@ Messages can also be seen by browsing to the source file in the Data Sources tre \image html messages_datasource_tree.png -*/ */ diff --git a/docs/doxygen-user/command_line_ingest.dox b/docs/doxygen-user/command_line_ingest.dox index bb248220c9..bdb466dfc6 100644 --- a/docs/doxygen-user/command_line_ingest.dox +++ b/docs/doxygen-user/command_line_ingest.dox @@ -2,7 +2,7 @@ \section command_line_ingest_overview Overview -The Command Line Ingest feature allows you to process a \ref ds_page "data source" with Autopsy from the command line. Autopsy will automatically create a case with the settings you specify and will generate a \ref report_case_uco report. +The Command Line Ingest feature allows you to run many of Autopsy's functions from the command line. You can add data sources to cases, choose which ingest modules to run, and automatically generate a report. When complete, these cases can be opened as normal or you can simply use the reports and other output without opening Autopsy. \section command_line_ingest_config Configuration @@ -10,39 +10,149 @@ Go to Tools->Options and then select the "Command Line Ingest" tab. \image html command_line_ingest_options.png -First, enter the output folder for the cases. Next, use the button to open the ingest module settings. Here you can configure the \ref ingest_page settings that will be used when running from the command line. +Use the ingest module settings to configure how you want to run ingest. This is the same as normal \ref ingest_page "ingest module" configuration - choose a file filter then enable or disable the individual ingest modules, changing their settings if desired. Press "OK" to save your settings. -\section command_line_ingest_run Running Autopsy +Use the report module settings to choose and configure a report type. Only the selected report type will be generated. Configuration is generally the same as normal \ref reporting_page "report generation" with some slight differences. This is mainly seen in places where your options are dependent on the open case, such as choosing \ref tagging_page "tags" to report on or \ref interesting_files_identifier_page "interesting file" set names to include. For example, the HTML report normally allows you to choose specific tags to include but for command line ingest it will only have the option to include all tags. + +\section command_line_ingest_commands Command Options In a command prompt, navigate to the Autopsy bin folder. This is normally located at "C:\Program Files\Autopsy-version\bin". \image html command_line_ingest_bin_dir.png -Now run autopsy with the following parameters, substituting the path to your data source and your desired case name. Both \ref ds_img "disk images" and \ref ds_log "logical files" are supported. Note that the case name must be unique for each run. +The table below shows a summary of the command line operations. You can run one or more at a time, though you must always either create a case or open an existing case. + +
+ + + + + + + + + + + + +
--listAllDataSources
+
OperationCommand(s)Parameter(s)Example
Create New Case
--createCase
--caseName
+--caseBaseDir
--createCase --caseName="test5" --caseBaseDir="C:\work\cases"
Open Existing Case 
--caseDir
--caseDir="C:\work\Cases\test5_2019_09_20_11_01_29"
Add a Data Source
--addDataSource
+--runIngest (optional)
--dataSourcePath
--addDataSource --dataSourcePath="R:\work\images\small2.img" --runIngest
Run Ingest on Existing Data Source
--runIngest
--dataSourceObjectId
--runIngest --dataSourceObjectId=1
Generate Reports
--generateReports
 
--generateReports
Create List of Data Sources
--listAllDataSources
 
+ + +More details on each operation along with additional examples are given below. + +\subsection command_line_cases Creating and Opening Cases + +You will always need to either create a case or give the path to an existing case. When creating a case, the current timestamp will be added to the case name. For example, running this command: \verbatim -autopsy64.exe --inputPath=(data source path) --caseName=(case name) --runFromCommandLine=true +autopsy64.exe --createCase --caseName="test5" --caseBaseDir="C:\work\cases" \endverbatim -In the example below, we're going to process a disk image with path "R:\work\images\xp-sp3-v4.E01" and name the case "xpCase". +could create a case folder "test5_2019_09_20_11_01_29". Note that even though a timestamp is added to the name, the --caseName field must be unique for each run. + +\image html command_line_ingest_case_folder.png + +Once a case is created you will need to use the full path to the case instead of the case name and base folder. For example, if we created the empty case "test5" as above, we could use the following command to add a data source to it: + +\verbatim +autopsy64.exe --caseDir="C:\work\Cases\test5_2019_09_20_11_01_29" --addDataSource + --dataSourcePath="R:\work\images\small2.img" +\endverbatim + +\subsection command_line_ds Adding a New Data Source and Running Ingest + +You can add a data source to a new case or an existing case using the --addDataSource option and then giving the path to the data source. If you use the --runIngest option, the ingest modules you selected in the \ref command_line_ingest_config "configuration step" will be run on the data source. Both \ref ds_img "disk images" and \ref ds_log "logical files" are supported. You can only add one data source at a time. + +In this example, we'll create a new case named "test6" and add the data source "blue_images.img". + +\verbatim +autopsy64.exe --createCase --caseName="test6" --caseBaseDir="C:\work\cases" --addDataSource + --dataSourcePath="R:\work\images\blue_images.img" +\endverbatim + +And here we'll add another data source ("green_images.img") to the case we just made and run ingest on it. Note that ingest will only run on the new data source ("green_images.img"), not the one already in the case ("blue_images.img"). + +\verbatim +autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --addDataSource --runIngest + --dataSourcePath="R:\work\images\green_images.img" +\endverbatim + +Finally we'll add a folder ("Test files") as a logical file set to a new case ("test9"). + +\verbatim +autopsy64.exe --createCase --caseName="test9" --caseBaseDir="C:\work\Cases" --addDataSource + --dataSourcePath="R:\work\images\Test files" --runIngest +\endverbatim + +\subsection command_line_existing_ds Running Ingest on an Existing Data Source + +You can run ingest on a data source already in the case if you know its object ID. To find this, go to the case folder and open the "Command Output" folder. + +\image html command_line_ingest_output_folder.png + +If you've run with the --listAllDataSources option, there will be at least one file starting "listAllDataSources". Open the most recent one - the format will be similar to this: + +\verbatim +{ + "@dataSourceName" : "blue_images.img", + "@dataSourceObjectId" : "1" +} { + "@dataSourceName" : "green_images.img", + "@dataSourceObjectId" : "84" +} +\endverbatim + +You can also look through the addDataSource files to find the one corresponding to the file you want to ingest. The format will be the same. Once you know the data source object ID, you can use the --dataSourceObjectId option to specify it. For example, this will run ingest on "blue_images.img": + +\verbatim +autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --runIngest --dataSourceObjectId=1 +\endverbatim + +\subsection command_line_report Generating Reports + +You can generate a report on the case using the --generateReports option. You can select which report type to export through the Autopsy options panel (see the \ref command_line_ingest_config "configuration section"). This option can be run alone or at the same time as you're processing a data source. In this example we're adding a new data source ("small2.img") and generating a report. + +\verbatim +autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --addDataSource + --dataSourcePath="R:\work\images\small2.img" --runIngest --generateReports +\endverbatim + +\subsection command_line_listds Listing All Data Sources + +You can add the --listAllDataSources at any time to output a list of all data sources currently in the case along with their object IDs, to be used when \ref command_line_existing_ds "running on an existing data source". This command can even be run alone with just the path to the case. + +\verbatim +autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --listAllDataSources +\endverbatim + + +\section command_line_ingest_run Running Autopsy + +Once you determine which parameters you need, it's time to run Autopsy. In the example below we're creating a new case ("xpCase"), adding a data source to it ("xp-sp3-v4.001"), running ingest and generating a report. The report type was \ref command_line_ingest_config "configured" earlier to be an HTML report. \image html command_line_ingest_command_entry.png -You'll start seeing output in the command prompt and the Autopsy UI will open. In the middle of the UI you'll see the following dialog: +If you've entered everything correctly, Autopsy will load and you'll see this dialog in the middle of the screen: \image html command_line_ingest_dialog.png -Once Autopsy finishes processing you'll be back at the command window. Press enter to return to the command prompt. +If you instead see the normal case open dialog, it most likely means that your command line is malformed. Verify that there are no typos and that you have the appropriate parameters for the operation(s) you're attempting. + +If everything works correctly, you'll see a log of the processing being done and Autopsy will close when finished. \image html command_line_ingest_console_output.png + \section command_line_ingest_results Viewing Results You can open the case created on the command line like any other Autopsy case. Simply go to "Open Case" and then browse to the output folder you set up in the \ref command_line_ingest_config section and look for the folder starting with your case name. It will have a timestamp appended to the name you specified. \image html command_line_ingest_open_case.png -If you are only interested in the \ref report_case_uco report then you don't need to open Autopsy. The report can be found in the case folder under "Reports\CASE-UCO" and then an automatically generated data source name containing the ID and timestamp. +If you are only interested in the reports then you don't need to open Autopsy. You can just browse to the "Reports" folder in the case and access the reports directly. \image html command_line_ingest_report.png diff --git a/docs/doxygen-user/communications.dox b/docs/doxygen-user/communications.dox index 94b46e9d30..9efd0f02f1 100644 --- a/docs/doxygen-user/communications.dox +++ b/docs/doxygen-user/communications.dox @@ -14,7 +14,7 @@ The Communications Visualization Tool is loaded through the Tools->Communication \image html cvt_main.png -From the left hand column, you can choose which devices to display, which types of data to display, and optionally select a time range. You can also choose to limit the display to only the most recent communications. After any changes to the filters, use the Apply button to update the tables. +From the left hand column, you can choose which devices to display, which types of data to display, and optionally select a time range. You can also choose to limit the display to only the most recent communications. After any changes to the filters, use the Apply button to update the tables. You can hide this column by clicking the left arrow at the top of the column. The middle column displays each account, its device and type, and the number of associated messages (emails, call logs, etc.). By default it will be sorted in descending order of frequency. The middle column and the right hand column both have a \ref ui_quick_search feature which can be used to quickly find a visible item in their section's table. @@ -25,7 +25,7 @@ Selecting an account in the middle column will bring up the data for that accoun \image html cvt_summary_tab.png -
  • The Messages tab displays any messages or call logs associated with the account. The Messages will either be in a thread, or listed under a node called "Unthreaded". Clicking on the "Unthreaded" node will show all the messages that are not "Threaded". Call logs will all be under a node named "Call Logs". +
  • The Messages tab displays any messages or call logs associated with the account. The Messages will either be in a thread, or listed under a node called "Unthreaded". Clicking on the "Unthreaded" node will show all the messages that are not "Threaded". \image html cvt_messages_threaded.png @@ -37,11 +37,15 @@ If the message has attachments, you can view them on the Attachments tab. If you \image html cvt_message_attach.png +
  • The Call Logs tab shows all call log entries involving the selected account. + +\image html cvt_call_log.png +
  • The Contacts tab shows any information on this account that was found in a contacts file. \image html cvt_contacts.png -
  • The Media tab shows thumbnails of any media files in messages for that account. If you click on one, it will show the message the media file came from. +
  • The Media Attachments tab shows thumbnails of any media files in messages for that account. If you click on one, it will show the message the media file came from. \image html cvt_media.png diff --git a/docs/doxygen-user/email_parser.dox b/docs/doxygen-user/email_parser.dox index ce3796bd43..517ccb069d 100644 --- a/docs/doxygen-user/email_parser.dox +++ b/docs/doxygen-user/email_parser.dox @@ -3,7 +3,7 @@ What Does It Do ======== -The Email Parser module identifies Thunderbird MBOX files and PST format files based on file signatures, extracting the e-mails from them, adding the results to the Blackboard. This module skips known files and creates a Blackboard artifact for each message. It adds email attachments as derived files. +The Email Parser module identifies MBOX, EML and PST format files based on file signatures, extracting the e-mails from them, adding the results to the Blackboard. This module skips known files and creates a Blackboard artifact for each message. It adds email attachments as derived files. This allows the user to identify email-based communications from the system being analyzed. diff --git a/docs/doxygen-user/images/LogicalImager/VHDfolder.png b/docs/doxygen-user/images/LogicalImager/VHDfolder.png new file mode 100644 index 0000000000..edd4b3e0ce Binary files /dev/null and b/docs/doxygen-user/images/LogicalImager/VHDfolder.png differ diff --git a/docs/doxygen-user/images/LogicalImager/configure_drive.png b/docs/doxygen-user/images/LogicalImager/configure_drive.png index 94b03beeed..186f98310e 100644 Binary files a/docs/doxygen-user/images/LogicalImager/configure_drive.png and b/docs/doxygen-user/images/LogicalImager/configure_drive.png differ diff --git a/docs/doxygen-user/images/LogicalImager/dsp_select.png b/docs/doxygen-user/images/LogicalImager/dsp_select.png index e05f677af8..e40d980044 100644 Binary files a/docs/doxygen-user/images/LogicalImager/dsp_select.png and b/docs/doxygen-user/images/LogicalImager/dsp_select.png differ diff --git a/docs/doxygen-user/images/LogicalImager/fileTree.png b/docs/doxygen-user/images/LogicalImager/fileTree.png new file mode 100644 index 0000000000..dd0ba5175f Binary files /dev/null and b/docs/doxygen-user/images/LogicalImager/fileTree.png differ diff --git a/docs/doxygen-user/images/LogicalImager/interestingFiles.png b/docs/doxygen-user/images/LogicalImager/interestingFiles.png new file mode 100644 index 0000000000..f961eca707 Binary files /dev/null and b/docs/doxygen-user/images/LogicalImager/interestingFiles.png differ diff --git a/docs/doxygen-user/images/LogicalImager/main_config_panel.png b/docs/doxygen-user/images/LogicalImager/main_config_panel.png index 6e3c1f15ce..a1b40a0fa4 100644 Binary files a/docs/doxygen-user/images/LogicalImager/main_config_panel.png and b/docs/doxygen-user/images/LogicalImager/main_config_panel.png differ diff --git a/docs/doxygen-user/images/LogicalImager/nonVHDexport.png b/docs/doxygen-user/images/LogicalImager/nonVHDexport.png new file mode 100644 index 0000000000..5e655a39a9 Binary files /dev/null and b/docs/doxygen-user/images/LogicalImager/nonVHDexport.png differ diff --git a/docs/doxygen-user/images/LogicalImager/nonVHDfolder.png b/docs/doxygen-user/images/LogicalImager/nonVHDfolder.png new file mode 100644 index 0000000000..1042e5a13b Binary files /dev/null and b/docs/doxygen-user/images/LogicalImager/nonVHDfolder.png differ diff --git a/docs/doxygen-user/images/LogicalImager/save.png b/docs/doxygen-user/images/LogicalImager/save.png new file mode 100644 index 0000000000..6b9281081a Binary files /dev/null and b/docs/doxygen-user/images/LogicalImager/save.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_bin_dir.png b/docs/doxygen-user/images/command_line_ingest_bin_dir.png index 6ca5d2fe87..5f23d867e1 100644 Binary files a/docs/doxygen-user/images/command_line_ingest_bin_dir.png and b/docs/doxygen-user/images/command_line_ingest_bin_dir.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_case_folder.png b/docs/doxygen-user/images/command_line_ingest_case_folder.png new file mode 100644 index 0000000000..e156cf5688 Binary files /dev/null and b/docs/doxygen-user/images/command_line_ingest_case_folder.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_command_entry.png b/docs/doxygen-user/images/command_line_ingest_command_entry.png index c346aa937a..2f19ffa098 100644 Binary files a/docs/doxygen-user/images/command_line_ingest_command_entry.png and b/docs/doxygen-user/images/command_line_ingest_command_entry.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_console_output.png b/docs/doxygen-user/images/command_line_ingest_console_output.png index ad1756ceea..a17285db1b 100644 Binary files a/docs/doxygen-user/images/command_line_ingest_console_output.png and b/docs/doxygen-user/images/command_line_ingest_console_output.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_open_case.png b/docs/doxygen-user/images/command_line_ingest_open_case.png index f95b008ef9..2a216bae1b 100644 Binary files a/docs/doxygen-user/images/command_line_ingest_open_case.png and b/docs/doxygen-user/images/command_line_ingest_open_case.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_options.png b/docs/doxygen-user/images/command_line_ingest_options.png index f654290ab2..015c2dee45 100644 Binary files a/docs/doxygen-user/images/command_line_ingest_options.png and b/docs/doxygen-user/images/command_line_ingest_options.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_output_folder.png b/docs/doxygen-user/images/command_line_ingest_output_folder.png new file mode 100644 index 0000000000..36ca54b99e Binary files /dev/null and b/docs/doxygen-user/images/command_line_ingest_output_folder.png differ diff --git a/docs/doxygen-user/images/command_line_ingest_report.png b/docs/doxygen-user/images/command_line_ingest_report.png index 9b26ccbd45..13070eeccd 100644 Binary files a/docs/doxygen-user/images/command_line_ingest_report.png and b/docs/doxygen-user/images/command_line_ingest_report.png differ diff --git a/docs/doxygen-user/images/cvt_call_log.png b/docs/doxygen-user/images/cvt_call_log.png new file mode 100644 index 0000000000..3b06328687 Binary files /dev/null and b/docs/doxygen-user/images/cvt_call_log.png differ diff --git a/docs/doxygen-user/images/cvt_contacts.png b/docs/doxygen-user/images/cvt_contacts.png index d085873a16..9d86b82ccc 100644 Binary files a/docs/doxygen-user/images/cvt_contacts.png and b/docs/doxygen-user/images/cvt_contacts.png differ diff --git a/docs/doxygen-user/images/cvt_main.png b/docs/doxygen-user/images/cvt_main.png index 56d565bab3..1c9ec9e903 100644 Binary files a/docs/doxygen-user/images/cvt_main.png and b/docs/doxygen-user/images/cvt_main.png differ diff --git a/docs/doxygen-user/images/cvt_media.png b/docs/doxygen-user/images/cvt_media.png index 5fbc03d785..bf4982cb62 100644 Binary files a/docs/doxygen-user/images/cvt_media.png and b/docs/doxygen-user/images/cvt_media.png differ diff --git a/docs/doxygen-user/images/cvt_message_email.png b/docs/doxygen-user/images/cvt_message_email.png index 335a90013b..e322a3548b 100644 Binary files a/docs/doxygen-user/images/cvt_message_email.png and b/docs/doxygen-user/images/cvt_message_email.png differ diff --git a/docs/doxygen-user/images/cvt_messages_threaded.png b/docs/doxygen-user/images/cvt_messages_threaded.png index 4399f8d3a6..6ca9274426 100644 Binary files a/docs/doxygen-user/images/cvt_messages_threaded.png and b/docs/doxygen-user/images/cvt_messages_threaded.png differ diff --git a/docs/doxygen-user/images/cvt_summary_tab.png b/docs/doxygen-user/images/cvt_summary_tab.png index c5e634c8a0..53f4ee148c 100644 Binary files a/docs/doxygen-user/images/cvt_summary_tab.png and b/docs/doxygen-user/images/cvt_summary_tab.png differ diff --git a/docs/doxygen-user/images/log4j.PNG b/docs/doxygen-user/images/log4j.PNG index 15a4ba1113..c9800a1fe4 100644 Binary files a/docs/doxygen-user/images/log4j.PNG and b/docs/doxygen-user/images/log4j.PNG differ diff --git a/docs/doxygen-user/images/photorec_output.PNG b/docs/doxygen-user/images/photorec_output.PNG index 168e8b0272..566e6a4652 100644 Binary files a/docs/doxygen-user/images/photorec_output.PNG and b/docs/doxygen-user/images/photorec_output.PNG differ diff --git a/docs/doxygen-user/images/portable_case_folder.png b/docs/doxygen-user/images/portable_case_folder.png index a02bd4e087..1e3a5b98df 100644 Binary files a/docs/doxygen-user/images/portable_case_folder.png and b/docs/doxygen-user/images/portable_case_folder.png differ diff --git a/docs/doxygen-user/images/timeline_counts_view.png b/docs/doxygen-user/images/timeline_counts_view.png new file mode 100644 index 0000000000..0f14c75c79 Binary files /dev/null and b/docs/doxygen-user/images/timeline_counts_view.png differ diff --git a/docs/doxygen-user/images/timeline_details_view.png b/docs/doxygen-user/images/timeline_details_view.png new file mode 100644 index 0000000000..7357191a97 Binary files /dev/null and b/docs/doxygen-user/images/timeline_details_view.png differ diff --git a/docs/doxygen-user/images/timeline_list_view.png b/docs/doxygen-user/images/timeline_list_view.png new file mode 100644 index 0000000000..46e012a315 Binary files /dev/null and b/docs/doxygen-user/images/timeline_list_view.png differ diff --git a/docs/doxygen-user/logical_imager.dox b/docs/doxygen-user/logical_imager.dox index 5f46068418..51729020e3 100644 --- a/docs/doxygen-user/logical_imager.dox +++ b/docs/doxygen-user/logical_imager.dox @@ -4,15 +4,17 @@ The logical imager allows you to collect files from a live Windows computer. The imager is configured with rules that specify what files to collect. Rules can be based on file attributes such as folder names, extensions, and sizes. You can use this feature when you do not have time or authorization to perform a full drive acquisition. -The logical imager produces one or more sparse VHD images that contain all of the file system data that was read. These VHD images can be imported into Autopsy or mounted by Windows. The imager also enumerates the user accounts on the system and can generate alerts if encryption programs exist. +Logical imager can save the matching files in two ways. The default method is to save individual files, which is the faster method and uses less disk space. The other option is to produce one or more sparse VHD images that contain all of the file system data that was read. These VHD images can be imported into Autopsy or mounted by Windows. In either case, the logical imager also enumerates the user accounts on the system and can generate alerts if encryption programs exist. The general workflow is:
    • Configure logical imager using Autopsy. This will copy a configuration file specifying which files to collect and the logical imager executable to the target drive. -
    • Insert the drive into the target system and run logical imager. This will give you a folder containing the sparse VHD copy of the target system (or multiple VHDs if more than one drive was analyzed), a file containing user account information, and a record of which files generated alerts. +
    • Insert the drive into the target system and run logical imager. This will give you a folder containing either the matching files or one or more sparse VHDs, a file containing user account information, and a record of which files generated alerts.
    • Load the result of running logical imager into Autopsy to browse any matching files and see user account information.
    +Currently logical imager can only be configured on Windows, and will only analyze Windows systems. You will also need to be able to run logical imager as administrator on the target system. + \section logical_imager_config Configuration To start, open Autopsy and go to Tools->Create Logical Imager. @@ -22,7 +24,7 @@ To start, open Autopsy and go to Tools->Create Logical Imager.
    • Configuring an external drive -The normal use case is to select a drive from the list under "Configure selected external drive." This will put the logical imager executable and a configuration file into the root directory of that drive once you finish the configuration. It is important to run the executable from the root of your drive because its presence on the drive makes the imager skip that drive during processing. MOVE THIS?? +The normal use case is to select a drive from the list under "Configure selected external drive." This will put the logical imager executable and a configuration file into the root directory of that drive once you finish the configuration. Note that logical imager can only be configured and run on a non-FAT drive (except exFAT) due to the 4 GB max file size on FAT systems. \image html LogicalImager/configure_drive.png @@ -38,10 +40,32 @@ In either case you can now configure your imager. If the configuration file alre \image html LogicalImager/main_config_panel.png -On the left side you can see each rule in the configuration file. Each of these rules will be applied against the live system. A rule has a name, an optional description, one or more conditions, and settings for what should happen when a file matching the rule is found. When you select a rule you'll see all the settings for that rule on the right side of the panel. You can edit or delete rules once you select them. There are also two global settings in the bottom right that apply to the configuration file as a whole: +On the left side you can see each rule in the configuration file. Each of these rules will be applied against the live system. A rule has a name, an optional description, one or more conditions, and settings for what should happen when a file matching the rule is found. When you select a rule you'll see all the settings for that rule on the right side of the panel. You can edit or delete rules once you select them. There are also global settings in the bottom right that apply to the configuration file as a whole:
        -
      • Alert if encryption programs are found - This will add a predefined rule to find encryption programs and alert and export any that are found. You will not be able to edit this rule. -
      • Continue imaging after searches are performed - By default, the logical imager will only copy sectors that it uses or that are part of matching files being exported. If this option is selected, logical imager will go back through the image after the rule matching is complete and copy over any remaining sectors. This will take longer to run and result in much larger VHD images. +
      • Alert if encryption programs are found - This will add a predefined rule to find encryption programs and alert and export any that are found. You will not be able to edit this rule. + + +
      • Prompt before exiting imager - If selected, you will have to press a key at the end of the logical imager run. This keeps the command prompt window open so you can look at the output. +
      • Create VHD - If selected, a sparse VHD will be created while running logical imager. See more details below. +
          +
        • Continue imaging after searches are performed - Only relevant when creating a VHD. By default, the logical imager will only copy sectors that it uses or that are part of matching files being exported. If this option is selected, logical imager will go back through the image after the rule matching is complete and copy over any remaining sectors. This will take longer to run and result in much larger VHD images. +
        +
      + +More information on creating a VHD versus saving any matching files directly: +
        +
      • Non-VHD mode +
          +
        • In this mode, any files matching a rule that has "Extract File" enable will be copied to the logical imager output folder. The paths and names are shortened in the output folder but will appear in their original form once the results are loaded into Autopsy. +
        • Pros: Faster than creating a VHD and will typically use significantly less disk space +
        • Cons: Not all file metadata is preserved; no additional data about the file system is saved +
        +
      • VHD mode +
          +
        • In this mode, all data read by the logical imager is copied into a VHD. This will include any matching files in their entirety, and also metadata for all files on the system. +
        • Pros: More complete metadata for the matching files, contains information about the system beyond the matching files. Have the option to copy the entire file system. +
        • Cons: Slower and uses more disk space. Can also be confusing in Autopsy because many file entries will have no data (their metadata was copied to the VHD but not their contents) +
      To make a new rule, click on the "New Rule" button. @@ -57,7 +81,7 @@ There are two rule types to choose from: For either rule type, you start by entering a rule name and optional description. You will also need to choose at least one action to take when a match is found.
      • Alert in Imager console if rule matches - this will display the file data in the console and add it to the "alerts.txt" output file. -
      • Extract file if it matches a rule - this will ensure that the matching file's contents will be copied to the sparse VHD +
      • Extract file if it matches a rule - this will ensure that the matching file's contents will be copied to the output folder or sparse VHD
      Attribute rules can have one or more conditions. All conditions must be true for a rule to match. @@ -76,6 +100,10 @@ Full path rules have a single condition. \image html LogicalImager/full_path_rule.png +Once you've set up all your rules, go to the next panel and click "Save" to save your configuration file and the logical imager executable to the location you selected. + +\image html LogicalImager/save.png + \section logical_imager_running Running Logical Imager \subsection logical_imager_default_run Running with the Default Configuration @@ -84,9 +112,9 @@ Using the defaults in the configuration process will create a drive with the con \image html LogicalImager/exe_folder.png -The default case is to run the logical imager on every drive except the one containing it. Note that the logical imager executable must be in the root directory for the drive to be skipped. To run the imager, right-click on "tsk_logical_imager.exe" and select "Run as administrator". This will open a console window where you'll see some information about the processing and if you set any rules to create alerts, you'll see matches in the console window as well. The window will close automatically when the processing is complete. +The default case is to run the logical imager on every drive except the one containing it. Note that the logical imager executable must be in the root directory for the drive to be skipped. To run the imager, right-click on "tsk_logical_imager.exe" and select "Run as administrator". This will open a console window where you'll see some information about the processing and if you set any rules to create alerts, you'll see matches in the console window as well. Depending on which option you selected during configuration, the window may close automatically when the processing is complete. -The logical imager will start writing the sparse VHD(s) and any other data to a directory next to the executable. +The logical imager will start writing to a directory next to the executable. \image html LogicalImager/output_folder.png @@ -106,7 +134,29 @@ If you want to specify the drive to run on, you can use the "-i" flag. This can \section logical_imager_results Viewing Results -The logical imager results can be added to an Autopsy case as a \ref ds_page "data source". This brings in the sparse VHD(s) as a disk image and also adds the other files created by the logical imager. Select the "Autopsy Imager" option and proceed to the next page. +\subsection logical_imager_folder Output folder structure + +If logical imager was run in the default mode (not creating a VHD), the output folder will look similar to this: + +\image html LogicalImager/nonVHDfolder.png + +Folder contents: +
        +
      • root folder containing any extracted files. These can be viewed directly in Windows explorer but since the names have been changed and directories flattened, it is better to view them in Autopsy. +\image html LogicalImager/nonVHDexport.png +
      • config.json is a copy of the configuration file used to generate the output +
      • console.txt is a copy of everything written to the Windows console +
      • SearchResults.txt is used when \ref logical_imager_dsp "adding the results to Autopsy" to match up the exported files with their original paths and file names, and the rule that they matched. This file will be added to the Autopsy case as a \ref reporting_page "report". +
      • users.txt contains information on user accounts found in the system. It will also be added to the Autopsy case as a \ref reporting_page "report". +
      + +If logical imager was set to create VHDs, you'll see those VHDs in the output folder (along with the other output files described above except the root folder): + +\image html LogicalImager/VHDfolder.png + +\subsection logical_imager_dsp Adding results to Autopsy + +The logical imager results can be added to an Autopsy case as a \ref ds_page "data source". This brings in either just the matching files or the sparse VHD(s) as a disk image, and also adds the other files created by the logical imager. Select the "Autopsy Imager" option and proceed to the next page. \image html LogicalImager/dsp_select.png @@ -116,7 +166,15 @@ In the top section, you can see all the logical imager result folders in the roo If your logical imager results are in a different location, select "Manually Choose Folder" and use the "Browse" button to locate your results. -In either case you'll get to configure the \ref ingest_page "ingest modules" to run. You can run any of them, but since your disk image may not be complete you may see more errors than normal. For example, the sparse VHD may contain the entire file allocation table but the actual data that goes with the files will be missing. +In either case you'll get to configure the \ref ingest_page "ingest modules" to run. You can run any of them, but if you created a VHD your disk image may not be complete you may see more errors than normal. For example, the sparse VHD will contain the entire file allocation table but the actual data that goes with most of the files will be missing. + +Regardless of whether you used a VHD or not, the matching files will appear in their original path with their original name in the \ref tree_viewer_page. If you did not create a VHD, you will only see matching files in the tree. If you did create a VHD, you'll see entries for non-matching files as well, though the contents of these files may not exist. + +\image html LogicalImager/fileTree.png + +Interesting File artifacts will be made for any files that match the rules. + +\image html LogicalImager/interestingFiles.png The alert and user files created by the logical imager can be found under the Reports section of the Tree Viewer. diff --git a/docs/doxygen-user/multi-user/installSolr.dox b/docs/doxygen-user/multi-user/installSolr.dox index 9ef8e3c907..3d55f29efc 100644 --- a/docs/doxygen-user/multi-user/installSolr.dox +++ b/docs/doxygen-user/multi-user/installSolr.dox @@ -66,13 +66,16 @@ The following steps will configure Solr to run using an account that will have a \image html transientcache.PNG

      4. Log Configuration: Edit "C:\Bitnami\solr-4.10.3-0\apache-solr\resources/log4j.properties" to configure Solr log settings: - - Increase the log rotation size threshold (_log4j\.appender\.file\.MaxFileSize_) from 4MB to 100MB. - - Remove the _CONSOLE_ appender from the _log4j\.rootLogger_ line. -

      - The log file should end up looking like this (modified lines are highlighted in yellow -

      - \image html log4j.PNG -

      +
        +
      1. Increase the log rotation size threshold (_log4j\.appender\.file\.MaxFileSize_) from 4MB to 100MB. +
      2. Remove the _CONSOLE_ appender from the _log4j\.rootLogger_ line. +
      3. Add the line "log4j.logger.org.apache.solr.update.processor.LogUpdateProcessor=WARN". +
      +The log file should end up looking like this (modified lines are highlighted in yellow + +\image html log4j.PNG + + 5. Schema Configuration: From an Autopsy installation, copy the following into "C:\Bitnami\solr-4.10.3-0\apache-solr\solr": - The folder "C:\Program Files\Autopsy-XXX(current version)\autopsy\solr\solr\configsets" - The folder "C:\Program Files\Autopsy-XXX(current version)\autopsy\solr\solr\lib" diff --git a/docs/doxygen-user/portable_case.dox b/docs/doxygen-user/portable_case.dox index b476206a38..de90443026 100644 --- a/docs/doxygen-user/portable_case.dox +++ b/docs/doxygen-user/portable_case.dox @@ -2,7 +2,7 @@ \section portable_case_overview Overview -A portable case is a partial copy of a normal Autopsy case that can be opened from anywhere. It contains a subset of the data from its original case and has been designed to make it easy to share relevant data with other examiners. +A portable case is a partial copy of a normal Autopsy case that can be opened from anywhere. It contains a subset of the data from its original case and has been designed to make it easy to share relevant data with other examiners. Portable cases will contain a \ref report_case_uco report detailing the contents of the portable case. The general use case is as follows: @@ -29,17 +29,17 @@ A portable case can contain tagged files and results and data from the Interesti \image html portable_case_interesting_items.png -You can tag any additional files you want to include in the portable case. See the \ref tagging_page page for details on how to create tags. You can see what tags you've added in the \ref tree_viewer_page. +You can tag any additional files you want to include in the portable case. See the \ref tagging_page page for details on how to create tags. Note that the outlines from \ref image_tagging "image tags" will also be visible in the portable case. You can see what tags you've added in the \ref tree_viewer_page. \image html portable_case_tags.png -Portable cases are created through the \ref reporting_page feature. The Generate Report dialog will display a list of all tags and interesting file sets that are in use in the current case and you can choose which ones you would like to include. At the bottom you can choose to optionally package the case. Choosing to package the case without chunking will simply compress the portable case in a single archive that can be extracted with common compression programs. If you choose split the packaged case into multiple files, you will need to use the "Unpackage Portable Case" option before loading it. This will be discussed in the next section. +Portable cases are created through the \ref reporting_page feature. The Generate Report dialog will display a list of all tags and interesting file sets that are in use in the current case and you can choose which ones you would like to include. At the bottom you can choose to optionally package the case. Choosing to package the case without chunking will simply compress the portable case in a single archive that can be extracted with common compression programs. If you choose split the packaged case into multiple files, you will need to use the "Unpack and Open Portable Case" option to open it. This will be discussed in the next section. The portable case will be placed in the "Reports" folder in the current case. \image html portable_case_report_panel.png -Here you can see an unpackaged portable case. It will be named with the original case name plus "(Portable)". The portable case is initially missing many of the normal Autopsy folders - these will be created the first time a user opens it. +Here you can see an unpackaged portable case. It will be named with the original case name plus "(Portable)". The portable case is initially missing many of the normal Autopsy folders - these will be created the first time a user opens it. It will however start with a "Reports" folder that contains an automatically generated \ref report_case_uco report. \image html portable_case_folder.png @@ -49,7 +49,7 @@ If you packaged the portable case but did not choose to split it into chunks, yo \section portable_case_usage Using a Portable Case -If your portable case was packaged, you'll first need to unpackage it. Open the "Case" menu and then select "Unpackage Portable Case". This will bring up a dialog where you can browse to your packaged case and select where to extract it to. Once unpackaged you can open it normally. +Unpackaged portable cases can be opened like any other case through Case->Open Case. If your portable case was packaged, you'll need to use the unpack option to open it. Open the "Case" menu and then select "Unpack and Open Portable Case" option. This will bring up a dialog where you can browse to your packaged case and select where to extract it to. The case will also open. Note that any changes made to the case at this point will be saved to the unpacked location, and next time you open it you will need to browse to the unpacked folder. \image html portable_case_unpackage.png diff --git a/docs/doxygen-user/timeline.dox b/docs/doxygen-user/timeline.dox index d55f9a43bf..3ad6de8d86 100644 --- a/docs/doxygen-user/timeline.dox +++ b/docs/doxygen-user/timeline.dox @@ -1,37 +1,46 @@ /*! \page timeline_page Timeline -Overview -======== -This document outlines the use of the Timeline feature of Autopsy. This feature was funded by DHS S&T to help provide free and open source digital forensics tools to law enforcement. -This document assumes basic familiarity with Autopsy. -Quick Start -=========== --# Create a case as normal and add a disk image (or folder of files) as a data source. To get the most out of the timeline, ensure that you have the hash lookup module enabled with NSRL (to ignore known files) and have the EXIF and recent activity modules enabled to collect additional temporal data. --# After the image has been added, click "Tools", "Timeline" in the menu. This will open the Timeline tool in a new window. You can do this while ingest is running, but you will not have access to the temporal data that will be found after you create the timeline, unless you re-open the timeline tool. - +\section timeline_overview Overview +This document outlines the use of the Timeline feature of Autopsy. This feature was funded by DHS S&T to help provide free and open source digital forensics tools to law enforcement. The timeline feature can help answer questions such as these: - - -Use Case Details -================ -- In addition to the basic ideas presented in the previous section, here are some hints on use cases that were designed into the tool. - When did major web activity occur on a system? - When were external devices plugged into the system? - When were pictures with EXIF information added? - What websites were accessed that resulted in file system modifications immediately after? +Note that as of Autopsy 4.13, timeline events are now generated during ingest and stored in the case database instead of a separate database. For this reason, older cases will not longer work with timeline. + +\section timeline_quickstart Quick Start + +Use this section to learn the basics of timeline. More details on the display options can be found later in this document. + +First you'll need to have a case open in Autopsy. To get the most out of timeline, you'll want to do the following during ingest: +- Enable the \ref hash_db_page "hash lookup module" and use the NSRL to ignore known files +- Enable the \ref recent_activity_page "recent activity module" to generate web-related events and other miscellaneous event types +- Enable the \ref EXIF_parser_page "EXIF parser module" to generate events on when images were +- Enable other ingest modules that apply to your data. If you have email data, ensure the \ref email_parser_page "email parser module" is enabled. If you are analyzing mobile devices, ensure the \ref android_analyzer_page "Android analyzer module" and any other relevant modules are enabled. + +To open timeline, either use the "Timeline" button or navigate to "Tools" then "Timeline" in the menu. You can open timeline while an image is processing but the data will not be complete until it finishes. Timeline will start in \ref timeline_counts_view "counts view" with a chart showing the number of events in each time period. + +\image html timeline_counts_view.png + +You can click on one of the segments of the graphs to see the list of events in the lower left. Clicking on a single event will display details in the lower right section. + +You can change the view mode using the buttons in the upper middle area of the window. The second view mode, \ref timeline_details_view "details view", shows information on events that happened in a specific time period. This mode is best used after filtering down to a small window of time. + +\image html timeline_details_view.png + +The final view mode is the \ref timeline_list_view "list view". This view shows every event in the order it occurred. This can be helpful to see which other events happened in the same time frame as an event of interest. As with the details mode, this mode is best used with filters to reduce the number events shown. + +\image html timeline_list_view.png - - -Basic Concepts -============== +\section timeline_basic_concepts Basic Concepts This section covers some basic concepts of the interface. +\subsection timeline_events Events -Events ------- The timeline tool is organized around events. An Event has a timestamp, a type, and a description. Note: all Events are discrete, but might be grouped together to form clusters with a duration in the Details View depending on the level of Description that is enabled in the UI. The timeline collects data from multiple sources and organizes the events into the following taxonomy: @@ -47,6 +56,8 @@ The timeline collects data from multiple sources and organizes the events into t - Web Bookmarks (creation) - Web History - Web Searches + - Web Form Auto Fill + - Web Form Address - Miscellaneous - Messages - GPS Routes @@ -57,28 +68,30 @@ The timeline collects data from multiple sources and organizes the events into t - Installed Programs - Exif metadata - Devices Attached + - Log Entry + - Registry +\subsection timeline_viz_types Visualization Types +There are three different graph types that the Autopsy viewer provides. Each is better suited for a different type of question that the investigator is trying to answer. You can change between the three types in top part of the interface. -Visualization Types ------------ -There are two different graph types that the Autopsy viewer provides. Each is better suited for a different type of question that the investigator is trying to answer. You can change between the two types in top part of the interface (see previous section for a screen shot). +The \ref timeline_counts_view "Counts View" shows a stacked bar chart. Use this type of graph to show how much activity occurred in a given time frame. It won’t show you specific events though. It can be helpful to determine when the computer was last used or how often it was used. When you open a timeline, it will open in this style of graph. -The __Counts View__ shows a stacked bar chart. Use this type of graph to show how much activity occurred in a given time frame. It won’t show you specific events though. It can be helpful to determine when the computer was last used or how often it was used. When you open a timeline, it will open in this style of graph. +The \ref timeline_details_view "Details View" shows individual or groups of related events. Date/time is represented horizontally along the x-axis, but the vertical axis does not represent any specific units. You would use this interface to answer questions about what specific events happened in a given time frame or what events occurred before or after a given event. You would generally use this type of interface after using the Counts View to identify a period of time that you wanted details on. There can be a lot of details in this view and we have introduced zooming concepts, as described in the next section, to help with this. -The __Details View__ shows individual or groups of related events. Date/time is represented horizontally along the x-axis, but the vertical axis does not represent any specific units. You would use this interface to answer questions about what specific events happened in a given time frame or what events occurred before or after a given event. You would generally use this type of interface after using the Counts View to identify a period of time that you wanted details on. There can be a lot of details in this view and we have introduced zooming concepts, as described in the next section, to help with this. +The \ref timeline_list_view "List View" shows all events in the order they occurred. This can be useful for knowing what happened before and after a certain event. For example, if you have a web download, you can find out other files that were created before or after that. The list view can be over whelming because there can be thousands of events in any given time range. Use the filters described below to bring the number of events down to a relevant size. The table on the bottom left hand side of the panel has a \ref ui_quick_search feature which can be used to quickly find a node in the table. -Visualization settings ----------------------- +\subsection timeline_viz_settings Visualization settings + The toolbar above the visualization area shows settings specific to the active visualization. These settings affect the way events are displayed and/or the layout of the visualization. -Zooming -------- +\subsection timeline_zooming Zooming + A common challenge with timeline analysis is information overload. To help with this, the Autopsy interface has three ways of zooming that will help you identify the correct data. These can be controlled from a single area in the upper left of the interface. @@ -89,19 +102,17 @@ A common challenge with timeline analysis is information overload. To help with For the quick start approach to things, you should keep this in mind: Double clicking on something will change only one of these levels of zooming. We have tried to choose what would be most intuitive for most use cases. If you want to choose a different zooming approach, use the sliders in the upper left or right click on the chart. -History -------- +\subsection timeline_history History + If at any time you want to back out to something you saw before, use the back and forward history buttons in the upper left , or the keyboard shortcut `Alt + Left/Right`. -Timeline Interaction and Configuration Details -====================================================== +\section timeline_interaction Timeline Interaction and Configuration Details -Filters / Events ----------------- +\subsection timeline_filters Filters / Events This area allows the user to apply filters to limit what events are shown in the visualization. When the Details View is active, a tab in this area also enables navigating the visualization by event descriptions ( see the Details View section for more on this) @@ -114,8 +125,7 @@ The __Event Types__ filter allows the user to select which event types should be The Event Type hierarchy displayed in the filter tab also functions as the __legend__ for the visualizations. Events are color-coded to match their type, and have the corresponding icon displayed in several places. -Time Range Selection ---------------------- +\subsection timeline_time_range Time Range Selection The time range selection area provides several means of adjusting the displayed time range. Date/Time fields show the exact date and time of the start(left) and end(right) of the displayed range. The user can type directly into these fields or use a graphical date/time chooser to modify the start or end time. @@ -123,18 +133,17 @@ The minus and plus hour glass buttons(/) zoom the visible time range out and in The last method to adjust the visible time range is via the range slider. The user can position each end independently to adjust the start and end time respectively or drag the highlighted blue section to move the visible range without changing its length. In both visualizations, the user can also right-drag (starting in empty space) a time span, represented by a pale blue box, and then double click it to zoom the visible time range. Right clicking the blue time span box clears it. -Histogram ---------- +\subsection timeline_histogram Histogram Behind the time range slider is a histogram of all events in the case. The histogram can help to put the main visualization in perspective by showing a high level summary of all events in the case, with a representation of the visible time range superimposed via the time range slider. The histogram divides the entire time span of all events in the case into equal intervals and shows the number of events in each interval via the height of the corresponding bar. The histogram should only be used for relative comparison and context and not for determining exact numbers or times of events. Note: This histogram is not affected by filters or zooming. -Time Zone ----- +\subsection timeline_time_zone Time Zone + The user can choose between viewing events in their local time zone or in Universal Coordinated Time. -Visualization Area: Counts View -------------------------------- +\subsection timeline_counts_view Visualization Area: Counts View + The Counts View shows a stacked bar chart with time periods along the x-axis and event counts along the y-axis. The height of each bar represents the number of events that occurred in that time period. The different colored segments represent different event types. Right clicking the bars brings up a context menu with selection and zooming actions. The only setting specific to the Counts View is what kind of vertical scale to use: The linear scale is good for many use cases. When this scale is selected, the height of the bars represents the counts in a linear, one-to-one fashion, and the y-axis is labeled with values. When the range of values is very large, time periods with low counts may have a bar that is too small to see. To help the user detect this, the labels for date ranges with events are bold. To see bars that are too small, there are three options: adjust the window size so that the visualization area has more vertical space, adjust the time range shown so that time periods with larger bars are excluded, or adjust the scale setting to logarithmic. @@ -142,9 +151,8 @@ The only setting specific to the Counts View is what kind of vertical scale to u The logarithmic scale represents the number of events in a non-linear way that compresses the difference between large and small numbers. Note that even with the logarithmic scale, an extremely large difference in counts may still produce bars too small to see. In this case the only option may be to filter events to reduce the difference in counts. NOTE: Because the logarithmic scale is applied to each event type separately, the meaning of the height of the combined bar is not intuitive, and to emphasize this, no labels are shown on the y-axis with the logarithmic scale. The logarithmic scale should be used to quickly compare the counts *across time within a type, or across types for one time period, but not both.* The actual counts (available in tooltips or the result viewer) should be used for absolute comparisons. Use the logarithmic scale with care. +\subsection timeline_details_view Visualization Area: Details View -Visualization Area: Details View ---------------------------------- The Details View shows events clustered by their description. Date/time is represented horizontally along the x-axis, but the vertical axis does not represent anything and is only used as a space to layout overlapping events. Events with the same type and description that occur close together in time may be clustered together. The Time Unit, Event Type and Description Detail sliders control how events are clustered. When the Description Detail level is at full, it is likely that very few events will be clustered, resulting in an enormous amount of detail being displayed. This can cause significant UI lag, and so __it is not recommended to use the full description unless the time range has been narrowed and/or filters applied to reduced the number of events shown__. Projections of the selected clusters are displayed on the x-axis to help visualize the temporal relationships between them. @@ -158,16 +166,11 @@ __Truncate Descriptions__: The user can select ‘truncate descriptions’ and __Description Visibility__: The user may choose a description visibility level of ‘show’, ‘counts only’, or ‘hide’. Show is the default. If Counts only is selected, only the count in parenthesis is shown, if hide is selected the entire text label is hidden. Counts only and hide are useful if the user wants to get a less cluttered view, focussed more on when event clusters occurred and their type, and is not interested in the descriptions. -Clicking the small green [+] button in a cluster will expand it with the next level of detail. The events in the cluster will be displayed clustered at a time scale appropriate for their extent and the detail level chosen. This can be repeated for the subclusters, to create a nested hierarchy of clusters. Clicking the red [-] button collapses a cluster to a lower level of detail. As with the global description level, care should be used when fully expanding large clusters, as this may cause an enormous amount of detail to be shown, slowing the tool down. - - - - -When the Detail View is active, the Events tab next to the Filters tab is enabled. This tab shows a list of all the descriptions presented in the visualization. Selecting a description in the list highlights all the event clusters with that description. - - +Clicking the small green [+] button in a cluster will expand it with the next level of detail. The events in the cluster will be displayed clustered at a time scale appropriate for their extent and the detail level chosen. This can be repeated for the subclusters, to create a nested hierarchy of clusters. Clicking the red [-] button collapses a cluster to a lower level of detail. As with the global description level, care should be used when fully expanding large clusters, as this may cause an enormous amount of detail to be shown, slowing the tool down. +\subsection timeline_list_view Visualization Area: List View +The List View shows all of the events in your selected time range. You can control this time range using the "Start" and "End" entries below the list, or by moving the endpoints of the blue line that is is displayed over the bar graph under the list. Selecting an event in the list will display its details in the bottom section of the screen. */ diff --git a/docs/doxygen/Doxyfile b/docs/doxygen/Doxyfile index bf66159b48..b640c41549 100644 --- a/docs/doxygen/Doxyfile +++ b/docs/doxygen/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = "Autopsy" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 4.12.0 +PROJECT_NUMBER = 4.13.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears a the top of each page and should give viewer a @@ -1063,7 +1063,7 @@ GENERATE_HTML = YES # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_OUTPUT = api-docs/4.12.0/ +HTML_OUTPUT = api-docs/4.13.0/ # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). diff --git a/docs/doxygen/debugTsk.dox b/docs/doxygen/debugTsk.dox index d70c014667..5e8ab961b2 100644 --- a/docs/doxygen/debugTsk.dox +++ b/docs/doxygen/debugTsk.dox @@ -3,7 +3,7 @@ If you find that you need to debug some of the C/C++ code from The Sleuth Kit (TSK), then here are the steps to follow: -# Ensure that you have the Debug version of the TSK JNI dll built (both 32-bit and 64-bit to be safe). This assumes you built TSK from source and are not simply using the developer platform. You may have to build the libtsk_jni twice because sommetimes it complains about not being able to find a .map file. --# Run the 'dist-debug' target for the TSK DataModel project. This copies the debug versions of the dll into the JAR file. If you run the 'dist' target, then you will get Release versions of the dll and you won't have the needed symbols for debugging. +-# Run the 'Debug-PostgreSQL' target for the TSK DataModel project. This copies the debug versions of the dll into the JAR file. If you run the 'dist' target, then you will get Release versions of the dll and you won't have the needed symbols for debugging. -# Build the Autopsy suite so that it copies the new JAR file with the debug dlls. -# Set your breakpoints in the TSK source. -# Run Autopsy in the debugger. diff --git a/docs/doxygen/modIngest.dox b/docs/doxygen/modIngest.dox index ff4f051479..a2d81627b9 100644 --- a/docs/doxygen/modIngest.dox +++ b/docs/doxygen/modIngest.dox @@ -192,33 +192,29 @@ The first question that you must answer is what type of data do you want the use -# Data that is in a big text file or some other report that the user can review. To do this, you will use the Case.addReport() method to make the output available in the directory tree. -\subsection ingest_modules_making_results_bb Posting Results to the Blackboard +\subsection ingest_modules_making_results_bb Saving Results to the Blackboard The blackboard is used to store results so that they are displayed in the results tree. -See \ref platform_blackboard for details on posting results to it. You use the blackboard when you have specific items to show the user. if you want to just shown them a big report from another library or tool, see \ref mod_report_page. +See \ref platform_blackboard for details on saving results to it. You use the blackboard when you have specific items to show the user. If you want to just shown them a big report from another library or tool, see \ref mod_report_page. The blackboard defines artifacts for specific data types (such as web bookmarks). You can use one of the standard artifact types or create your own. -When modules add data to the blackboard, they should notify listeners of the new -data by invoking the org.sleuthkit.autopsy.ingest.IngestServices.fireModuleDataEvent() method. -Do so as soon as you have added an artifact to the blackboard. -This allows other modules (and the main UI) to know when to query the blackboard -for the latest data. However, if you are writing a large number of blackboard -artifacts in a loop, it is better to invoke org.sleuthkit.autopsy.ingest.IngestServices.fireModuleDataEvent() -only once after the bulk write, so as not to flood the system with events. +After you've added an artifact and all of its attributes to the blackboard, you should call sleuthkit.Blackboard.postArtifact(), which will: +
        +
      • Analyze the artifact and add any timestamps to the Timeline tables +
      • Send an event over the Sleuth Kit event bus that the artifact(s) was added +
          +
        • Autopsy is a listener of this event bus and will rebroadcast the event to other Autopsy modules +
        • Keyword search also listens for this event and will index the artifact +
        +
      -Further, when modules create artifacts, they should be indexed for keyword search, -using the method org.sleuthkit.autopsy.casemodule.services.Blackboard.indexArtifact(BlackboardArtifact artifact). This can be done -in the following way: - -\code -Blackboard blackboard = Case.getCurrentCase().getServices().getBlackboard(); -try { - blackboard.indexArtifact(artifact); //Your artifact as the argument. -} -catch (BlackboardException ex) { - //YOUR EXCEPTION BEHAVIOR HERE. -} -\endcode +This means you no longer have to make separate calls to: + - Index the artifact + - Fire the event to refresh the UI. + +If you are creating a large number of artifacts, you may see better performance if you save all the artifacts you create and do one bulk post at the end using sleuthkit.Blackboard.postArtifacts(). You can also post batches of artifacts instead of saving all of them until the end. + +You should not be using the Autopsy version of Blackboard. Those methods have all been deprecated and is another example of us moving "services" into the TSK data model. \subsection ingest_modules_making_results_report Making a Report diff --git a/docs/doxygen/modMobile.dox b/docs/doxygen/modMobile.dox index ac24ad8fff..255638aa48 100644 --- a/docs/doxygen/modMobile.dox +++ b/docs/doxygen/modMobile.dox @@ -13,6 +13,56 @@ The ingest module has a basic flow of The BlackBoard has standard artifacts for the standard cell phone forensics data types, such as BlackboardArtifact.TSK_CALLLOG. +There are a couple of classes that can help streamline processing mobile databases. The org.sleuthkit.autopsy.coreutils.AppSQLiteDB class has methods for opening and querying SQLite databases. For example, to find and open a database named "transfer20.db" in the "com.dewmobile.kuaiya.play" package, you can simply use the following method (examples in Python): +\verbatim +transferDbs = AppSQLiteDB.findAppDatabases(dataSource, "transfer20.db", True, "com.dewmobile.kuaiya.play") +\endverbatim + +Once you have your databases, you can run easily run queries on them: +\code +queryString = "SELECT device, name, direction, createtime, path, title FROM transfer" +transfersResultSet = transferDb.runQuery(queryString) +\endcode + +You can make Blackboard Artifacts using the org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper class. This gives you methods to make contacts, messages, and call log entries. The following is sample code in Python to set up the CommunicationArtifactsHelper and then use it to make artifacts. + +\code +transferDbHelper = CommunicationArtifactsHelper(current_case.getSleuthkitCase(), + "Zapya Analyzer", transferDb.getDBFile(), + Account.Type.ZAPYA) +\endcode + +\code +direction = CommunicationDirection.UNKNOWN +fromAddress = None +toAddress = None + +if (transfersResultSet.getInt("direction") == 1): + direction = CommunicationDirection.OUTGOING + toAddress = Account.Address(transfersResultSet.getString("device"), transfersResultSet.getString("name") ) +else: + direction = CommunicationDirection.INCOMING + fromAddress = Account.Address(transfersResultSet.getString("device"), transfersResultSet.getString("name") ) + +msgBody = "" # there is no body. +attachments = [transfersResultSet.getString("path")] +msgBody = general.appendAttachmentList(msgBody, attachments) + +timeStamp = transfersResultSet.getLong("createtime") / 1000 +messageArtifact = transferDbHelper.addMessage( + self._MESSAGE_TYPE, + direction, + fromAddress, + toAddress, + timeStamp, + MessageReadStatus.UNKNOWN, + None, # subject + msgBody, + None ) # thread id +\endcode + +Look in the autopsy\\InternalPythonModules\\android\\ folder for additional examples. + \section mod_mobile_std Android Module Autopsy comes with an Android module, as defined in various classes in the org.sleuthkit.autopsy.modules.android package. You can use those classes as a reference example. diff --git a/docs/doxygen/modReport.dox b/docs/doxygen/modReport.dox index 6199ce7560..5799314acf 100644 --- a/docs/doxygen/modReport.dox +++ b/docs/doxygen/modReport.dox @@ -58,7 +58,7 @@ As when generating table module reports, Autopsy will iterate through a list of \subsection report_create_module_general Creating a General Report Module -If you implement GeneralReportModule, the overriden methods will be: +If you implement GeneralReportModule, the overridden methods will be: - org.sleuthkit.autopsy.report.GeneralReportModule.generateReport(String reportPath, ReportProgressPanel progressPanel) - org.sleuthkit.autopsy.report.GeneralReportModule.getConfigurationPanel() @@ -125,4 +125,47 @@ Report modules developed using Jython are installed in Autopsy by placing them i directory. A window into the python_modules directory can be opened through the Autopsy's Tools -> Python Plugins menu item. Create a folder in this directory and create or place your Python scripts in this folder. + +\subsection report_create_module_persistence Persisting your Report Module Configuration + +Both Java and Python report modules can have their configurations persisted to disk. In order to do so, the report module must do +two things: +
        +
      1. The report module must have a class that defines the configuration for the module (for example, org.sleuthkit.autopsy.report.modules.html.HTMLReportModuleSettings). This class +must implement the ReportModuleSettings interface. Note that the ReportModuleSettings interface extends Serializable, therefore the report +settings class must contain serialVersionUID variable: + +\code +class HTMLReportModuleSettings implements ReportModuleSettings { + + private static final long serialVersionUID = 1L; + + HTMLReportModuleSettings() { + } + + @Override + public long getVersionNumber() { + return serialVersionUID; + } +} +\endcode + +
      2. The report module implementation class (e.g. ReportHTML) must implement the following methods of the ReportModule interface: +- getDefaultConfiguration() +- getConfiguration() +- setConfiguration(ReportModuleSettings settings) +
      + +The use case scenario for this API when the configuration of a report module is occurring is as follows (using HTMLReport as example): +
        +
      1. User clicks "Generate Reports" button in the Autopsy UI. +
      2. Report Configuration UI attempts to load the persisted reporting configuration from disk. +
      3. For each existing report module, if a persisted ReportModuleSettings (i.e. HTMLReportModuleSettings) configuration exists, Configuration UI calls HTMLReport.setConfiguration() with the persisted settings; Otherwise the Configuration UI calls HTMLReport.getDefaultConfiguration(), then HTMLReport.setConfiguration(). +
      4. Configuration UI calls HTMLReport.getConfigurationPanel(). The report module loads the settings into its configuration panel and returns the panel. +
      5. Configuration UI presents the panel to the user. +
      6. User interacts with the panel to modify the current settings of the report module. +
      7. Configuration UI calls HTMLReport.getConfiguration() and obtains the latest report module configuration. +
      8. Configuration UI persists the module settings. +
      + */ diff --git a/docs/doxygen/platformConcepts.dox b/docs/doxygen/platformConcepts.dox index af202a1cba..23389d935b 100644 --- a/docs/doxygen/platformConcepts.dox +++ b/docs/doxygen/platformConcepts.dox @@ -52,7 +52,7 @@ The blackboard allows modules to communicate with each other and the UI. It has The blackboard is not unique to Autopsy. It is part of The Sleuth Kit datamodel and The Sleuth Kit Framework. In the name of reducing the amount of documentation that we need to maintain, we provide links here to those documentation sources. -- The Blackboard +- The Blackboard \subsection mod_dev_other_services Framework Services and Utilities diff --git a/nbproject/project.properties b/nbproject/project.properties index 5d59189544..8132096d48 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -4,7 +4,7 @@ app.title=Autopsy ### lowercase version of above app.name=${branding.token} ### if left unset, version will default to today's date -app.version=4.12.0 +app.version=4.13.0 ### build.type must be one of: DEVELOPMENT, RELEASE #build.type=RELEASE build.type=DEVELOPMENT diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index 5518c97512..baab8573df 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -430,8 +430,8 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info report_index = line.find('INSERT INTO "reports"') > -1 or line.find('INSERT INTO reports ') > -1 layout_index = line.find('INSERT INTO "tsk_file_layout"') > -1 or line.find('INSERT INTO tsk_file_layout ') > -1 data_source_info_index = line.find('INSERT INTO "data_source_info"') > -1 or line.find('INSERT INTO data_source_info ') > -1 - event_description_index = line.find('INSERT INTO "tsk_event_descriptions"') > -1 or line.find('INSERT INTO data_source_info ') > -1 - events_index = line.find('INSERT INTO "tsk_events"') > -1 or line.find('INSERT INTO data_source_info ') > -1 + event_description_index = line.find('INSERT INTO "tsk_event_descriptions"') > -1 or line.find('INSERT INTO tsk_event_descriptions ') > -1 + events_index = line.find('INSERT INTO "tsk_events"') > -1 or line.find('INSERT INTO tsk_events ') > -1 ingest_job_index = line.find('INSERT INTO "ingest_jobs"') > -1 or line.find('INSERT INTO ingest_jobs ') > -1 examiners_index = line.find('INSERT INTO "tsk_examiners"') > -1 or line.find('INSERT INTO tsk_examiners ') > -1 ig_groups_index = line.find('INSERT INTO "image_gallery_groups"') > -1 or line.find('INSERT INTO image_gallery_groups ') > -1 diff --git a/thirdparty/rr-full/plugins/shellbags.pl b/thirdparty/rr-full/plugins/shellbags.pl index 5b8587af38..d8e770471e 100644 --- a/thirdparty/rr-full/plugins/shellbags.pl +++ b/thirdparty/rr-full/plugins/shellbags.pl @@ -3,6 +3,10 @@ # RR plugin to parse (Vista, Win7/Win2008R2) shell bags # # History: +# 20190715 - updated to parse WPD devices better +# 20180702 - update to parseGUID function +# 20180117 - modification thanks to input/data from Mike Godfrey +# 20160706 - update # 20150325 - updated parsing based on input from Eric Zimmerman # 20140728 - updated shell item 0x01 parsing # 20131216 - updated to support shell item type 0x52 @@ -31,7 +35,7 @@ # Moore for writing the shell bag parser for Registry Decoder, as well as # assistance with some parsing. # -# License: GPL v3 +# # copyright 2015 Quantum Analytics Research, LLC # Author: H. Carvey, keydet89@yahoo.com #----------------------------------------------------------- @@ -47,12 +51,12 @@ my %config = (hive => "USRCLASS\.DAT", hasShortDescr => 1, hasDescr => 0, hasRefs => 0, - version => 20150325); + version => 20190715); sub getConfig{return %config} sub getShortDescr { - return "Shell/BagMRU traversal in Win7 USRCLASS.DAT hives"; + return "Shell/BagMRU traversal in Win7+ USRCLASS\.DAT hives"; } sub getDescr{} sub getRefs {} @@ -90,6 +94,7 @@ my %cp_guids = ("{bb64f8a7-bee7-4e1a-ab8d-7d8273f7fdb6}" => "Action Center", "{a3dd4f92-658a-410f-84fd-6fbbbef2fffe}" => "Internet Options", "{a304259d-52b8-4526-8b1a-a1d6cecc8243}" => "iSCSI Initiator", "{725be8f7-668e-4c7b-8f90-46bdb0936430}" => "Keyboard", + "{bf782cc9-5a52-4a17-806c-2a894ffeeac5}" => "Language Settings", "{e9950154-c418-419e-a90a-20c5287ae24b}" => "Location and Other Sensors", "{1fa9085f-25a2-489b-85d4-86326eedcd87}" => "Manage Wireless Networks", "{6c8eec18-8d75-41b2-a177-8831d59d2d50}" => "Mouse", @@ -178,8 +183,8 @@ sub pluginmain { my $class = shift; my $hive = shift; ::logMsg("Launching shellbags v.".$VERSION); - ::rptMsg("shellbags v.".$VERSION); # banner - ::rptMsg("(".getHive().") ".getShortDescr()."\n"); # banner + ::rptMsg("shellbags v.".$VERSION); + ::rptMsg("(".getHive().") ".getShortDescr()."\n"); my %item = (); my $reg = Parse::Win32Registry->new($hive); @@ -226,6 +231,13 @@ sub traverse { my $type = unpack("C",substr($values{$v},2,1)); +# DEBUG ------------------------------------------------ +# ::rptMsg($key->get_path()."\\".$v); +# ::rptMsg(sprintf "Type = 0x%x",$type); +# probe($values{$v}); +# ::rptMsg(""); +# DEBUG ------------------------------------------------ + # Need to first check to see if the parent of the item was a zip folder # and if the 'zipsubfolder' value is set to 1 if (exists ${$parent}{zipsubfolder} && ${$parent}{zipsubfolder} == 1) { @@ -245,6 +257,9 @@ sub traverse { # System Folder %item = parseSystemFolderEntry($values{$v}); } + elsif ($type == 0x2a) { + $item{name} = substr($values{$v},0x3,3); + } elsif ($type == 0x2e) { # Device %item = parseDeviceEntry($values{$v}); @@ -374,11 +389,10 @@ sub parseVariableEntry { # Ref: http://msdn.microsoft.com/en-us/library/aa965725(v=vs.85).aspx my $stuff = $segs{"{b725f130-47ef-101a-a5f1-02608c9eebac}"}; - my $tag = 1; + my $t = 1; my $cnt = 0x10; - while($tag) { + while($t) { my $sz = unpack("V",substr($stuff,$cnt,4)); - return %item unless (defined $sz); my $id = unpack("V",substr($stuff,$cnt + 4,4)); #-------------------------------------------------------------- # sub-segment types @@ -388,14 +402,14 @@ sub parseVariableEntry { # 0x0c - size #-------------------------------------------------------------- if ($sz == 0x00) { - $tag = 0; + $t = 0; next; } elsif ($id == 0x0a) { my $num = unpack("V",substr($stuff,$cnt + 13,4)); my $str = substr($stuff,$cnt + 13 + 4,($num * 2)); - $str =~ s/\x00//g; + $str =~ s/\00//g; $item{name} = $str; } $cnt += $sz; @@ -406,24 +420,41 @@ sub parseVariableEntry { elsif (substr($data,4,4) eq "AugM") { %item = parseFolderEntry($data); } +# Code for Windows Portable Devices +# Added 20190715 + elsif (parseGUID(substr($data,42,16)) eq "{27e2e392-a111-48e0-ab0c-e17705a05f85}") { + my ($n0, $n1, $n2) = unpack("VVV",substr($data,62,12)); + + my $n0_name = substr($data,0x4A,($n0 * 2)); + $n0_name =~ s/\00//g; + + my $n1_name = substr($data,(0x4A + ($n0 * 2)),($n1 * 2)); + $n1_name =~ s/\00//g; + + if ($n0_name eq "") { + $item{name} = $n1_name; + } + else { + $item{name} = $n0_name; + } + } # Following two entries are for Device Property data elsif ($tag == 0x7b || $tag == 0xbb || $tag == 0xfb) { my ($sz1,$sz2,$sz3) = unpack("VVV",substr($data,0x3e,12)); $item{name} = substr($data,0x4a,$sz1 * 2); - $item{name} =~ s/\x00//g; + $item{name} =~ s/\00//g; } elsif ($tag == 0x02 || $tag == 0x03) { my ($sz1,$sz2,$sz3,$sz4) = unpack("VVVV",substr($data,0x26,16)); $item{name} = substr($data,0x36,$sz1 * 2); - $item{name} =~ s/\x00//g; + $item{name} =~ s/\00//g; } elsif (unpack("v",substr($data,6,2)) == 0x05) { my $o = 0x26; my $t = 1; while ($t) { my $i = substr($data,$o,1); - return %item unless (defined $i); - if ($i =~ m/\x00/) { + if ($i =~ m/\00/) { $t = 0; } else { @@ -447,7 +478,7 @@ sub parseNetworkEntry { my %item = (); $item{type} = unpack("C",substr($data,2,1)); - my @n = split(/\x00/,substr($data,4,length($data) - 4)); + my @n = split(/\00/,substr($data,4,length($data) - 4)); $item{name} = $n[0]; return %item; } @@ -464,13 +495,13 @@ sub parseZipSubFolderItem { # Get the opened/accessed date/time $item{datetime} = substr($data,0x24,6); - $item{datetime} =~ s/\x00//g; + $item{datetime} =~ s/\00//g; if ($item{datetime} eq "N/A") { } else { $item{datetime} = substr($data,0x24,40); - $item{datetime} =~ s/\x00//g; + $item{datetime} =~ s/\00//g; my ($date,$time) = split(/\s+/,$item{datetime},2); my ($mon,$day,$yr) = split(/\//,$date,3); my ($hr,$min,$sec) = split(/:/,$time,3); @@ -483,9 +514,9 @@ sub parseZipSubFolderItem { my $sz2 = unpack("V",substr($data,0x58,4)); my $str1 = substr($data,0x5C,$sz *2) if ($sz > 0); - $str1 =~ s/\x00//g; + $str1 =~ s/\00//g; my $str2 = substr($data,0x5C + ($sz * 2),$sz2 *2) if ($sz2 > 0); - $str2 =~ s/\x00//g; + $str2 =~ s/\00//g; if ($sz2 > 0) { $item{name} = $str1."\\".$str2; @@ -548,10 +579,10 @@ sub parseURIEntry { my $sz = unpack("V",substr($data,0x2a,4)); my $uri = substr($data,0x2e,$sz); - $uri =~ s/\x00//g; + $uri =~ s/\00//g; my $proto = substr($data,length($data) - 6, 6); - $proto =~ s/\x00//g; + $proto =~ s/\00//g; $item{name} = $proto."://".$uri." [".gmtime($item{uritime})."]"; @@ -601,8 +632,8 @@ sub parseGUID { my $d3 = unpack("v",substr($data,6,2)); my $d4 = unpack("H*",substr($data,8,2)); my $d5 = unpack("H*",substr($data,10,6)); - my $guid = sprintf "{%08x-%x-%x-$d4-$d5}",$d1,$d2,$d3; - + my $guid = sprintf "{%08x-%04x-%04x-$d4-$d5}",$d1,$d2,$d3; + if (exists $cp_guids{$guid}) { return "CLSID_".$cp_guids{$guid}; } @@ -625,6 +656,10 @@ sub parseDeviceEntry { my $ofs = unpack("v",substr($data,4,2)); my $tag = unpack("V",substr($data,6,4)); +#----------------------------------------------------- +# DEBUG +# ::rptMsg("parseDeviceEntry, tag = ".$tag); +#----------------------------------------------------- if ($tag == 0) { my $guid1 = parseGUID(substr($data,$ofs + 6,16)); my $guid2 = parseGUID(substr($data,$ofs + 6 + 16,16)); @@ -632,13 +667,17 @@ sub parseDeviceEntry { } elsif ($tag == 2) { $item{name} = substr($data,0x0a,($ofs + 6) - 0x0a); - $item{name} =~ s/\x00//g; + $item{name} =~ s/\00//g; } else { my $ver = unpack("C",substr($data,9,1)); - + my $idx = unpack("C",substr($data,3,1)); + + if ($idx == 0x80) { + $item{name} = parseGUID(substr($data,4,16)); + } # Version 3 = XP - if ($ver == 3) { + elsif ($ver == 3) { my $guid1 = parseGUID(substr($data,$ofs + 6,16)); my $guid2 = parseGUID(substr($data,$ofs + 6 + 16,16)); $item{name} = $guid1."\\".$guid2 @@ -649,14 +688,11 @@ sub parseDeviceEntry { my $userlen = unpack("V",substr($data,30,4)); my $devlen = unpack("V",substr($data,34,4)); my $user = substr($data,0x28,$userlen * 2); - $user =~ s/\x00//g; + $user =~ s/\00//g; my $dev = substr($data,0x28 + ($userlen * 2),$devlen * 2); - $dev =~ s/\x00//g; + $dev =~ s/\00//g; $item{name} = $user; - } - elsif (unpack("C",substr($data,3,1)) == 0x80) { - $item{name} = parseGUID(substr($data,4,16)); - } + } # Version unknown else { $item{name} = "Device Entry - Unknown Version"; @@ -697,7 +733,7 @@ sub parseControlPanelEntry { # #----------------------------------------------------------- sub parseFolderEntry { - my $data = shift; + my $data = shift; my %item = (); $item{type} = unpack("C",substr($data,2,1)); @@ -726,85 +762,105 @@ sub parseFolderEntry { my @m = unpack("vv",substr($data,$ofs_mdate,4)); ($item{mtime_str},$item{mtime}) = convertDOSDate($m[0],$m[1]); -# Need to read in short name; nul-term ASCII -# $item{shortname} = (split(/\x00/,substr($data,12,length($data) - 12),2))[0]; - $ofs_shortname = $ofs_mdate + 6; - my $tag = 1; - my $cnt = 0; - my $str = ""; - while($tag) { - my $s = substr($data,$ofs_shortname + $cnt,1); - return %item unless (defined $s); - if ($s =~ m/\x00/ && ((($cnt + 1) % 2) == 0)) { - $tag = 0; - } - else { - $str .= $s; - $cnt++; - } - } -# $str =~ s/\x00//g; - my $shortname = $str; - my $ofs = $ofs_shortname + $cnt + 1; -# Read progressively, 1 byte at a time, looking for 0xbeef - $tag = 1; - $cnt = 0; - while ($tag) { - my $s = substr($data,$ofs + $cnt,2); - return %item unless (defined $s); - if (unpack("v",$s) == 0xbeef) { - $tag = 0; - } - else { - $cnt++; - } - } - $item{extver} = unpack("v",substr($data,$ofs + $cnt - 4,2)); -# printf "Version: 0x%x\n",$item{extver}; - $ofs = $ofs + $cnt + 2; +# DEBUG ------------------------------------------------ +# Added 20160706 based on sample data provided by J. Poling - @m = unpack("vv",substr($data,$ofs,4)); - ($item{ctime_str},$item{ctime}) = convertDOSDate($m[0],$m[1]); - $ofs += 4; - @m = unpack("vv",substr($data,$ofs,4)); - ($item{atime_str},$item{atime}) = convertDOSDate($m[0],$m[1]); - - my $jmp; - if ($item{extver} == 0x03) { - $jmp = 8; - } - elsif ($item{extver} == 0x07) { - $jmp = 26; - } - elsif ($item{extver} == 0x08) { - $jmp = 30; - } - elsif ($item{extver} == 0x09) { - $jmp = 34; - } - else {} - - if ($item{type} == 0x31 && $item{extver} >= 0x07) { - my @n = unpack("Vvv",substr($data,$ofs + 8, 8)); - if ($n[2] != 0) { - $item{mft_rec_num} = getNum48($n[0],$n[1]); - $item{mft_seq_num} = $n[2]; -# ::rptMsg("MFT: ".$item{mft_rec_num}."/".$item{mft_seq_num}); -# probe($data); - } - } - - $ofs += $jmp; - - $str = substr($data,$ofs,length($data) - 30); - my $longname = (split(/\x00\x00/,$str,2))[0]; - $longname =~ s/\x00//g; - - if ($longname ne "") { - $item{name} = $longname; + if (length($data) < 0x30) { +# start at offset 0xE, read in nul-term ASCII string (until "\00" is reached) + $ofs_shortname = 0xE; + my $tag = 1; + my $cnt = 0; + my $str = ""; + while($tag) { + my $s = substr($data,$ofs_shortname + $cnt,1); + if ($s =~ m/\00/) { + $tag = 0; + } + else { + $str .= $s; + $cnt++; + } + } + $item{name} = $str; } else { - $item{name} = $shortname; +# Need to read in short name; nul-term ASCII +# $item{shortname} = (split(/\00/,substr($data,12,length($data) - 12),2))[0]; + $ofs_shortname = $ofs_mdate + 6; + my $tag = 1; + my $cnt = 0; + my $str = ""; + while($tag) { + my $s = substr($data,$ofs_shortname + $cnt,1); + if ($s =~ m/\00/ && ((($cnt + 1) % 2) == 0)) { + $tag = 0; + } + else { + $str .= $s; + $cnt++; + } + } +# $str =~ s/\00//g; + my $shortname = $str; + my $ofs = $ofs_shortname + $cnt + 1; +# Read progressively, 1 byte at a time, looking for 0xbeef + my $tag = 1; + my $cnt = 0; + while ($tag) { + if (unpack("v",substr($data,$ofs + $cnt,2)) == 0xbeef) { + $tag = 0; + } + else { + $cnt++; + } + } + $item{extver} = unpack("v",substr($data,$ofs + $cnt - 4,2)); +# printf "Version: 0x%x\n",$item{extver}; + $ofs = $ofs + $cnt + 2; + + my @m = unpack("vv",substr($data,$ofs,4)); + ($item{ctime_str},$item{ctime}) = convertDOSDate($m[0],$m[1]); + $ofs += 4; + my @m = unpack("vv",substr($data,$ofs,4)); + ($item{atime_str},$item{atime}) = convertDOSDate($m[0],$m[1]); + + my $jmp; + if ($item{extver} == 0x03) { + $jmp = 8; + } + elsif ($item{extver} == 0x07) { + $jmp = 26; + } + elsif ($item{extver} == 0x08) { + $jmp = 30; + } + elsif ($item{extver} == 0x09) { + $jmp = 34; + } + else {} + + if ($item{type} == 0x31 && $item{extver} >= 0x07) { + my @n = unpack("Vvv",substr($data,$ofs + 8, 8)); + if ($n[2] != 0) { + $item{mft_rec_num} = getNum48($n[0],$n[1]); + $item{mft_seq_num} = $n[2]; +# ::rptMsg("MFT: ".$item{mft_rec_num}."/".$item{mft_seq_num}); +# probe($data); + } + } + + $ofs += $jmp; + + my $str = substr($data,$ofs,length($data) - 30); + my $longname = (split(/\00\00/,$str,2))[0]; + $longname =~ s/\00//g; + + if ($longname ne "") { + $item{name} = $longname; + } + else { + $item{name} = $shortname; + } } return %item; } @@ -855,9 +911,7 @@ sub parseFolderEntry2 { my $tag = 1; while ($tag) { - my $s = substr($data,$ofs,2); - return %item unless (defined $s); - if (unpack("v",$s) == 0xbeef) { + if (unpack("v",substr($data,$ofs,2)) == 0xbeef) { $tag = 0; } else { @@ -894,9 +948,9 @@ sub parseFolderEntry2 { # } # ::rptMsg(""); - $item{name} = (split(/\x00\x00/,$str,2))[0]; - $item{name} =~ s/\x13\x20/\x2D\x00/; - $item{name} =~ s/\x00//g; + $item{name} = (split(/\00\00/,$str,2))[0]; + $item{name} =~ s/\13\20/\2D\00/; + $item{name} =~ s/\00//g; return %item; } @@ -907,7 +961,7 @@ sub parseNetworkEntry { my $data = shift; my %item = (); $item{type} = unpack("C",substr($data,2,1)); - my @names = split(/\x00/,substr($data,5,length($data) - 5)); + my @names = split(/\00/,substr($data,5,length($data) - 5)); $item{name} = $names[0]; return %item; } @@ -919,9 +973,9 @@ sub parseDatePathItem { my $data = shift; my %item = (); $item{datestr} = substr($data,0x18,30); - my ($file,$dir) = split(/\x00\x00/,substr($data,0x44,length($data) - 0x44)); - $file =~ s/\x00//g; - $dir =~ s/\x00//g; + my ($file,$dir) = split(/\00\00/,substr($data,0x44,length($data) - 0x44)); + $file =~ s/\00//g; + $dir =~ s/\00//g; $item{name} = $dir.$file; return %item; } @@ -958,7 +1012,6 @@ sub shellItem0x52 { while ($tag) { $d = substr($data,0x32 + $cnt,2); - return %item unless (defined $d); if (unpack("v",$d) == 0) { $tag = 0; } @@ -967,7 +1020,7 @@ sub shellItem0x52 { $cnt += 2; } } - $item{name} =~ s/\x00//g; + $item{name} =~ s/\00//g; if ($item{subtype} < 3) { $ofs = 0x32 + $cnt + 2; @@ -977,7 +1030,7 @@ sub shellItem0x52 { } $sz = unpack("V",substr($data,$ofs,4)); $item{str} = substr($data,$ofs + 4,$sz * 2); - $item{str} =~ s/\x00//g; + $item{str} =~ s/\00//g; return %item; } @@ -1058,4 +1111,4 @@ sub getNum48 { } } -1; +1; \ No newline at end of file diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml index 2f4d0ad72d..52e915e2f3 100644 --- a/thunderbirdparser/nbproject/project.xml +++ b/thunderbirdparser/nbproject/project.xml @@ -36,7 +36,7 @@ 10 - 10.16 + 10.17 diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java index e0b50ffa8c..ebdf934509 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -44,9 +44,8 @@ class EMLParser extends MimeJ4MessageParser { String ext = abFile.getNameExtension(); boolean isEMLFile = ext != null && ext.equals("eml"); if (isEMLFile) { - isEMLFile = (new String(buffer)).contains("To:"); //NON-NLS + isEMLFile = (new String(buffer)).contains(":"); //NON-NLS } - return isEMLFile; } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java index 1114d71eba..229615b7b3 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.thunderbirdparser; import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -31,6 +32,7 @@ import org.apache.james.mime4j.dom.Body; import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; import org.apache.james.mime4j.dom.Multipart; +import org.apache.james.mime4j.dom.SingleBody; import org.apache.james.mime4j.dom.TextBody; import org.apache.james.mime4j.dom.address.AddressList; import org.apache.james.mime4j.dom.address.Mailbox; @@ -298,7 +300,14 @@ class MimeJ4MessageParser { logger.log(Level.SEVERE, Bundle.MimeJ4MessageParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS return; } - String filename = FileUtil.escapeFileName(e.getFilename()); + String filename = e.getFilename(); + + if (filename == null) { + filename = "attachment" + e.hashCode(); + logger.log(Level.WARNING, String.format("Attachment has no file name using '%s'", filename)); + } + + filename = FileUtil.escapeFileName(filename); // also had some crazy long names, so make random one if we get those. // also from Japanese image that had encoded name @@ -308,40 +317,25 @@ class MimeJ4MessageParser { String uniqueFilename = fileID + "-" + index + "-" + email.getSentDate() + "-" + filename; String outPath = outputDirPath + uniqueFilename; - EncodedFileOutputStream fos; - BinaryBody bb; - try { - fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1); - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS - return; - } - - try { - Body b = e.getBody(); - if (b instanceof BinaryBody) { - bb = (BinaryBody) b; - bb.writeTo(fos); - } else { - // This could potentially be other types. Only seen this once. - } - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS - return; - } finally { - try { - fos.close(); + + Body body = e.getBody(); + if (body instanceof SingleBody) { + try (EncodedFileOutputStream fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1)) { + ((SingleBody) body).writeTo(fos); } catch (IOException ex) { - logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS + logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS + return; } - } - - EmailMessage.Attachment attach = new EmailMessage.Attachment(); - attach.setName(filename); - attach.setLocalPath(relModuleOutputPath + uniqueFilename); - attach.setSize(new File(outPath).length()); - attach.setEncodingType(TskData.EncodingType.XOR1); - email.addAttachment(attach); + + EmailMessage.Attachment attach = new EmailMessage.Attachment(); + attach.setName(filename); + attach.setLocalPath(relModuleOutputPath + uniqueFilename); + attach.setSize(new File(outPath).length()); + attach.setEncodingType(TskData.EncodingType.XOR1); + email.addAttachment(attach); + } + + } /** diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java index 5256afcc4c..f747ea2d9f 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java @@ -135,7 +135,7 @@ class PstParser { Iterable iterable = null; try { - iterable = getEmaiMessageIterator(pstFile.getRootFolder(), "\\", fileID, true); + iterable = getEmailMessageIterator(pstFile.getRootFolder(), "\\", fileID, true); } catch (PSTException | IOException ex) { logger.log(Level.WARNING, String.format("Exception thrown while parsing fileID: %d", fileID), ex); } @@ -202,7 +202,7 @@ class PstParser { Iterable iterable = null; try { - iterable = getEmaiMessageIterator(pstFile.getRootFolder(), "\\", fileID, false); + iterable = getEmailMessageIterator(pstFile.getRootFolder(), "\\", fileID, false); } catch (PSTException | IOException ex) { logger.log(Level.WARNING, String.format("Exception thrown while parsing fileID: %d", fileID), ex); } @@ -228,7 +228,7 @@ class PstParser { * @throws PSTException * @throws IOException */ - private Iterable getEmaiMessageIterator(PSTFolder folder, String path, long fileID, boolean wholeMsg) throws PSTException, IOException { + private Iterable getEmailMessageIterator(PSTFolder folder, String path, long fileID, boolean wholeMsg) throws PSTException, IOException { Iterable iterable = null; if (folder.getContentCount() > 0) { @@ -239,7 +239,7 @@ class PstParser { List subFolders = folder.getSubFolders(); for (PSTFolder subFolder : subFolders) { String newpath = path + "\\" + subFolder.getDisplayName(); - Iterable subIterable = getEmaiMessageIterator(subFolder, newpath, fileID, wholeMsg); + Iterable subIterable = getEmailMessageIterator(subFolder, newpath, fileID, wholeMsg); if (subIterable == null) { continue; } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index f428b455cf..5c42269a8a 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -190,7 +190,23 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { switch( result) { case OK: - processEmails(parser.getPartialEmailMessages(), parser.getEmailMessageIterator(), abstractFile); + Iterator pstMsgIterator = parser.getEmailMessageIterator(); + if (pstMsgIterator != null) { + processEmails(parser.getPartialEmailMessages(), pstMsgIterator , abstractFile); + } else { + // sometimes parser returns ParseResult=OK but there are no messages + postErrorMessage( + NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg", + abstractFile.getName()), + NbBundle.getMessage(this.getClass(), + "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details")); + logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS + // delete the temp file + if (file.delete() == false) { + logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + } + return ProcessResult.ERROR; + } break; case ENCRYPT: @@ -219,6 +235,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details")); logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS + // delete the temp file + if (file.delete() == false) { + logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + } return ProcessResult.ERROR; } @@ -411,10 +431,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * appropriate artifacts and derived files. * * @param partialEmailsForThreading - * @param fileMessageIterator + * @param fullMessageIterator * @param abstractFile */ private void processEmails(List partialEmailsForThreading, Iterator fullMessageIterator, AbstractFile abstractFile) { + // Putting try/catch around this to catch any exception and still allow // the creation of the artifacts to continue. try{ diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index 84f4cd92c8..d662c5f5e6 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -224,8 +224,6 @@ final class VcardParser { if (!tskBlackboard.artifactExists(abstractFile, BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT, attributes)) { artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); artifact.addAttributes(attributes); - List blackboardArtifacts = new ArrayList<>(); - blackboardArtifacts.add(artifact); extractPhotos(vcard, abstractFile, artifact); @@ -388,8 +386,12 @@ final class VcardParser { */ private void addPhoneAttributes(Telephone telephone, AbstractFile abstractFile, Collection attributes) { String telephoneText = telephone.getText(); + if (telephoneText == null || telephoneText.isEmpty()) { - return; + telephoneText = telephone.getUri().getNumber(); + if (telephoneText == null || telephoneText.isEmpty()) { + return; + } } // Add phone number to collection for later creation of TSK_CONTACT. @@ -397,34 +399,40 @@ final class VcardParser { if (telephoneTypes.isEmpty()) { ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephone.getText(), BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, attributes); } else { - for (TelephoneType type : telephoneTypes) { - /* - * Unfortunately, if the types are lower-case, they don't - * get separated correctly into individual TelephoneTypes by - * ez-vcard. Therefore, we must read them manually - * ourselves. - */ - List splitTelephoneTypes = Arrays.asList( - type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); + TelephoneType type = telephoneTypes.get(0); + /* + * Unfortunately, if the types are lower-case, they don't + * get separated correctly into individual TelephoneTypes by + * ez-vcard. Therefore, we must read them manually + * ourselves. + */ + List splitTelephoneTypes = Arrays.asList( + type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); - for (String splitType : splitTelephoneTypes) { - String attributeTypeName = "TSK_PHONE_NUMBER_" + splitType; - try { - BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); - if (attributeType == null) { + if (splitTelephoneTypes.size() > 0) { + String splitType = splitTelephoneTypes.get(0); + String attributeTypeName = "TSK_PHONE_NUMBER"; + if (splitType != null && !splitType.isEmpty()) { + attributeTypeName = "TSK_PHONE_NUMBER_" + splitType; + } + + try { + BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); + if (attributeType == null) { + try{ // Add this attribute type to the case database. attributeType = tskCase.addArtifactAttributeType(attributeTypeName, BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, - String.format("Phone (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + }catch (TskDataException ex) { + attributeType = tskCase.getAttributeType(attributeTypeName); } - ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephone.getText(), attributeType, attributes); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } catch (TskDataException ex) { - logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); } + ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); } - } + } } } @@ -447,34 +455,36 @@ final class VcardParser { if (emailTypes.isEmpty()) { ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL, attributes); } else { - for (EmailType type : emailTypes) { - /* - * Unfortunately, if the types are lower-case, they don't - * get separated correctly into individual EmailTypes by - * ez-vcard. Therefore, we must read them manually - * ourselves. - */ - List splitEmailTypes = Arrays.asList( - type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); + EmailType type = emailTypes.get(0); /* + * Unfortunately, if the types are lower-case, they don't + * get separated correctly into individual EmailTypes by + * ez-vcard. Therefore, we must read them manually + * ourselves. + */ + List splitEmailTypes = Arrays.asList( + type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); - for (String splitType : splitEmailTypes) { - String attributeTypeName = "TSK_EMAIL_" + splitType; - try { - BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); - if (attributeType == null) { - // Add this attribute type to the case database. - attributeType = tskCase.addArtifactAttributeType(attributeTypeName, - BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, - String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); - } - ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } catch (TskDataException ex) { - logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } - } - } + if (splitEmailTypes.size() > 0) { + String splitType = splitEmailTypes.get(0); + String attributeTypeName = "TSK_EMAIL_" + splitType; + if(splitType.isEmpty()) { + attributeTypeName = "TSK_EMAIL"; + } + try { + BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); + if (attributeType == null) { + // Add this attribute type to the case database. + attributeType = tskCase.addArtifactAttributeType(attributeTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + } + ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } catch (TskDataException ex) { + logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } + } } } @@ -490,7 +500,11 @@ final class VcardParser { private void addPhoneAccountInstances(Telephone telephone, AbstractFile abstractFile, Collection accountInstances) { String telephoneText = telephone.getText(); if (telephoneText == null || telephoneText.isEmpty()) { - return; + telephoneText = telephone.getUri().getNumber(); + if (telephoneText == null || telephoneText.isEmpty()) { + return; + } + } // Add phone number as a TSK_ACCOUNT. diff --git a/unix_setup.sh b/unix_setup.sh index eecf1ed8c5..e055c2797b 100644 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -5,7 +5,7 @@ # NOTE: update_sleuthkit_version.pl updates this value and relies # on it keeping the same name and whitespace. Don't change it. -TSK_VERSION=4.6.7 +TSK_VERSION=4.7.0 # In the beginning...