codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-10-03 07:49:52 +00:00
Code Issues Releases Wiki Activity

Merge branch 'timeline' of https://github.com/sleuthkit/autopsy into timeline

Browse Source
This commit is contained in:
Tim McIver
2013-01-24 14:08:24 -05:00
parent 15e256ffe9 2fe79f6281
commit ebeebc821a
15 changed files with 139 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BUILDING.txt
+5 -3
View File
@@ -11,18 +11,20 @@ correct C libraries.
STEPS:
1) Get Java Setup
1a) Download and install 32-bit version of JDK (32-bit is currently
1a) Download and install 32-bit version of JDK version 1.7 (32-bit is currently
needed even if you have a 64-bit system).
1b) Ensure that JDK_HOME is set to the root JDK directory.
1b) Ensure that JDK_HOME is set to the root JDK directory.
1c) (optional) Download and install Netbeans IDE (http://netbeans.org/)
Note: Netbeans IDE is not required to build and run Autopsy,
but it is a recommended IDE to use for development of Autopsy modules.
1d) (opetional) If you are going to package Autopsy, then you'll also
1d) (optional) If you are going to package Autopsy, then you'll also
need to set JRE_HOME to the root JRE directory.
1e) (optional) For some Autopsy features to be functional, you need to add java executable to the system PATH.
2) Get Sleuth Kit Setup
Core/src/org/sleuthkit/autopsy/datamodel/AbstractContentChildren.java
-4
View File
@@ -137,10 +137,6 @@ abstract class AbstractContentChildren<T> extends Keys<T> {
return ee.new EmailExtractedRootNode();
}
@Override
public AbstractNode visit(Bookmarks bks) {
return bks.new BookmarksRootNode();
}
@Override
public AbstractNode visit(Tags t) {
Core/src/org/sleuthkit/autopsy/datamodel/AutopsyItemVisitor.java
-5
View File
@@ -33,7 +33,6 @@ public interface AutopsyItemVisitor<T> {
T visit(KeywordHits kh);
T visit(HashsetHits hh);
T visit(EmailExtracted ee);
T visit(Bookmarks bks);
T visit(Tags t);
T visit(Images i);
T visit(Views v);
@@ -88,10 +87,6 @@ public interface AutopsyItemVisitor<T> {
return defaultVisit(ee);
}
@Override
public T visit(Bookmarks bks) {
return defaultVisit(bks);
}
@Override
public T visit(Tags t) {
Core/src/org/sleuthkit/autopsy/datamodel/Bookmarks.java
+6 -3
View File
@@ -47,8 +47,11 @@ import org.sleuthkit.datamodel.TskCoreException;
* Bookmarks are specialized tags - TSK_TAG_NAME starts with File Bookmark or
* Result Bookmark
*
* @deprecated cosolidated under Tags
*
* TODO bookmark hierarchy support (TSK_TAG_NAME with slashes)
*/
@Deprecated
public class Bookmarks implements AutopsyVisitableItem {
public static final String NAME = "Bookmarks";
@@ -69,7 +72,7 @@ public class Bookmarks implements AutopsyVisitableItem {
@Override
public <T> T accept(AutopsyItemVisitor<T> v) {
return v.visit(this);
return null; //v.visit(this);
}
/**
@@ -111,7 +114,7 @@ public class Bookmarks implements AutopsyVisitableItem {
@Override
public <T> T accept(DisplayableItemNodeVisitor<T> v) {
return v.visit(this);
return null; // v.visit(this);
}
@Override
@@ -197,7 +200,7 @@ public class Bookmarks implements AutopsyVisitableItem {
@Override
public <T> T accept(DisplayableItemNodeVisitor<T> v) {
return v.visit(this);
return null; //v.visit(this);
}
@Override
Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java
-13
View File
@@ -18,8 +18,6 @@
*/
package org.sleuthkit.autopsy.datamodel;
import org.sleuthkit.autopsy.datamodel.Bookmarks.BookmarksNodeRoot;
import org.sleuthkit.autopsy.datamodel.Bookmarks.BookmarksRootNode;
import org.sleuthkit.autopsy.datamodel.EmailExtracted.EmailExtractedAccountNode;
import org.sleuthkit.autopsy.datamodel.EmailExtracted.EmailExtractedFolderNode;
import org.sleuthkit.autopsy.datamodel.EmailExtracted.EmailExtractedRootNode;
@@ -56,8 +54,6 @@ public interface DisplayableItemNodeVisitor<T> {
T visit(EmailExtractedRootNode eern);
T visit(EmailExtractedAccountNode eean);
T visit(EmailExtractedFolderNode eefn);
T visit(BookmarksRootNode bksrn);
T visit(BookmarksNodeRoot bksrn);
T visit(TagsRootNode bksrn);
T visit(TagsNodeRoot bksrn);
T visit(TagNodeRoot tnr);
@@ -201,15 +197,6 @@ public interface DisplayableItemNodeVisitor<T> {
return defaultVisit(ldn);
}
@Override
public T visit(BookmarksRootNode bksrn) {
return defaultVisit(bksrn);
}
@Override
public T visit(BookmarksNodeRoot bksnr) {
return defaultVisit(bksnr);
}
@Override
public T visit(TagsRootNode bksrn) {
Core/src/org/sleuthkit/autopsy/datamodel/ResultsNode.java
-1
View File
@@ -36,7 +36,6 @@ public class ResultsNode extends DisplayableItemNode {
new KeywordHits(sleuthkitCase),
new HashsetHits(sleuthkitCase),
new EmailExtracted(sleuthkitCase),
new Bookmarks(sleuthkitCase), //TODO move to the top of the tree
new Tags(sleuthkitCase) //TODO move to the top of the tree
)), Lookups.singleton(NAME));
setName(NAME);
Core/src/org/sleuthkit/autopsy/datamodel/RootContentChildren.java
-6
View File
@@ -70,16 +70,12 @@ public class RootContentChildren extends AbstractContentChildren<Object> {
//TODO check
case TSK_TAG_FILE:
if (o instanceof Bookmarks)
this.refreshKey(o);
if (o instanceof Tags)
this.refreshKey(o);
break;
//TODO check
case TSK_TAG_ARTIFACT:
if (o instanceof Bookmarks)
this.refreshKey(o);
if (o instanceof Tags)
this.refreshKey(o);
break;
@@ -96,8 +92,6 @@ public class RootContentChildren extends AbstractContentChildren<Object> {
this.refreshKey(o);
else if (o instanceof EmailExtracted)
this.refreshKey(o);
else if (o instanceof Bookmarks)
this.refreshKey(o);
else if (o instanceof Tags)
this.refreshKey(o);
else if (o instanceof ExtractedContent)
Core/src/org/sleuthkit/autopsy/datamodel/Tags.java
+51 -22
View File
@@ -33,6 +33,7 @@ import org.openide.nodes.ChildFactory;
import org.openide.nodes.Children;
import org.openide.nodes.Node;
import org.openide.nodes.Sheet;
import org.openide.util.Exceptions;
import org.openide.util.Lookup;
import org.openide.util.lookup.Lookups;
import org.sleuthkit.autopsy.casemodule.Case;
@@ -62,8 +63,10 @@ public class Tags implements AutopsyVisitableItem {
private SleuthkitCase skCase;
public static final String NAME = "Tags";
private static final String TAG_ICON_PATH = "org/sleuthkit/autopsy/images/tag-folder-blue-icon-16.png";
//bookmarks are specializations of tags
public static final String BOOKMARK_TAG_NAME = "Bookmark";
private static final String BOOKMARK_ICON_PATH = "org/sleuthkit/autopsy/images/star-bookmark-icon-16.png";
private Map<BlackboardArtifact.ARTIFACT_TYPE, Map<String, List<BlackboardArtifact>>> tags;
private static final String EMPTY_COMMENT = "";
Tags(SleuthkitCase skCase) {
@@ -105,9 +108,7 @@ public class Tags implements AutopsyVisitableItem {
for (BlackboardAttribute attribute : artifact.getAttributes()) {
if (attribute.getAttributeTypeID() == ATTRIBUTE_TYPE.TSK_TAG_NAME.getTypeID()) {
String tagName = attribute.getValueString();
if (tagName.equals(Bookmarks.BOOKMARK_TAG_NAME)) {
break; // Don't add bookmarks
} else if (artTags.containsKey(tagName)) {
if (artTags.containsKey(tagName)) {
List<BlackboardArtifact> artifacts = artTags.get(tagName);
artifacts.add(artifact);
} else {
@@ -263,12 +264,16 @@ public class Tags implements AutopsyVisitableItem {
public class TagNodeRoot extends DisplayableItemNode {
TagNodeRoot(BlackboardArtifact.ARTIFACT_TYPE tagType, String tagName, List<BlackboardArtifact> artifacts) {
super(Children.create(new Tags.TagsChildrenNode(tagType, artifacts), true), Lookups.singleton(tagName));
super(Children.create(new Tags.TagsChildrenNode(tagType, tagName, artifacts), true), Lookups.singleton(tagName));
super.setName(tagName);
super.setDisplayName(tagName + " (" + artifacts.size() + ")");
this.setIconBaseWithExtension(TAG_ICON_PATH);
if (tagName.equals(BOOKMARK_TAG_NAME)) {
this.setIconBaseWithExtension(BOOKMARK_ICON_PATH);
} else {
this.setIconBaseWithExtension(TAG_ICON_PATH);
}
}
@Override
@@ -312,10 +317,12 @@ public class Tags implements AutopsyVisitableItem {
private List<BlackboardArtifact> artifacts;
private BlackboardArtifact.ARTIFACT_TYPE tagType;
private String tagName;
private TagsChildrenNode(BlackboardArtifact.ARTIFACT_TYPE tagType, List<BlackboardArtifact> artifacts) {
private TagsChildrenNode(BlackboardArtifact.ARTIFACT_TYPE tagType, String tagName, List<BlackboardArtifact> artifacts) {
super();
this.tagType = tagType;
this.tagName = tagName;
this.artifacts = artifacts;
}
@@ -330,13 +337,20 @@ public class Tags implements AutopsyVisitableItem {
//create node with action
BlackboardArtifactNode tagNode = null;
String iconPath;
if (tagName.equals(BOOKMARK_TAG_NAME)) {
iconPath = BOOKMARK_ICON_PATH;
} else {
iconPath = TAG_ICON_PATH;
}
//create actions here where Tag logic belongs
//instead of DataResultFilterNode w/visitors, which is much less pluggable and cluttered
if (tagType.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_TAG_ARTIFACT)) {
//in case of result tag, add a action by sublcassing bb art node
//this action will be merged with other actions set DataResultFIlterNode
//otherwise in case of
tagNode = new BlackboardArtifactNode(artifact, TAG_ICON_PATH) {
tagNode = new BlackboardArtifactNode(artifact, iconPath) {
@Override
public Action[] getActions(boolean bln) {
//Action [] actions = super.getActions(bln); //To change body of generated methods, choose Tools | Templates.
@@ -357,13 +371,13 @@ public class Tags implements AutopsyVisitableItem {
};
} else {
//for file tag, don't subclass to add the additional actions
tagNode = new BlackboardArtifactNode(artifact, TAG_ICON_PATH);
tagNode = new BlackboardArtifactNode(artifact, iconPath);
}
//add some additional node properties
int artifactTypeID = artifact.getArtifactTypeID();
final String NO_DESCR = "no description";
if (artifactTypeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_TAG_ARTIFACT.getTypeID()) {
final String NO_DESCR = "no description";
BlackboardArtifact sourceResult = Tags.getArtifactFromTag(artifact.getArtifactID());
String resultType = sourceResult.getDisplayName();
@@ -376,6 +390,21 @@ public class Tags implements AutopsyVisitableItem {
tagNode.addNodeProperty(resultTypeProp);
}
try {
//add source path property
final AbstractFile sourceFile = skCase.getAbstractFileById(artifact.getObjectID());
final String sourcePath = sourceFile.getUniquePath();
NodeProperty sourcePathProp = new NodeProperty("Source File Path",
"Source File Path",
NO_DESCR,
sourcePath);
tagNode.addNodeProperty(sourcePathProp);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error getting a file from artifact to get source file path for a tag, ", ex);
}
return tagNode;
}
}
@@ -396,10 +425,10 @@ public class Tags implements AutopsyVisitableItem {
BlackboardAttribute attr1 = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TAG_NAME.getTypeID(),
"", tagName);
attrs.add(attr1);
if (comment != null && ! comment.isEmpty()) {
if (comment != null && !comment.isEmpty()) {
BlackboardAttribute attr2 = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT.getTypeID(),
"", comment);
"", comment);
attrs.add(attr2);
}
bookArt.addAttributes(attrs);
@@ -427,17 +456,17 @@ public class Tags implements AutopsyVisitableItem {
BlackboardAttribute attr1 = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TAG_NAME.getTypeID(),
"", tagName);
if (comment != null && ! comment.isEmpty()) {
if (comment != null && !comment.isEmpty()) {
BlackboardAttribute attr2 = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT.getTypeID(),
"", comment);
"", comment);
attrs.add(attr2);
}
BlackboardAttribute attr3 = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TAGGED_ARTIFACT.getTypeID(),
"", artifact.getArtifactID());
attrs.add(attr1);
attrs.add(attr3);
bookArt.addAttributes(attrs);
} catch (TskCoreException ex) {
@@ -452,7 +481,7 @@ public class Tags implements AutopsyVisitableItem {
* @param comment the bookmark comment
*/
public static void createBookmark(AbstractFile file, String comment) {
createTag(file, Bookmarks.BOOKMARK_TAG_NAME, comment);
createTag(file, Tags.BOOKMARK_TAG_NAME, comment);
}
/**
@@ -462,7 +491,7 @@ public class Tags implements AutopsyVisitableItem {
* @param comment the bookmark comment
*/
public static void createBookmark(BlackboardArtifact artifact, String comment) {
createTag(artifact, Bookmarks.BOOKMARK_TAG_NAME, comment);
createTag(artifact, Tags.BOOKMARK_TAG_NAME, comment);
}
/**
@@ -474,7 +503,7 @@ public class Tags implements AutopsyVisitableItem {
try {
Case currentCase = Case.getCurrentCase();
SleuthkitCase skCase = currentCase.getSleuthkitCase();
return skCase.getBlackboardArtifacts(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TAG_NAME, Bookmarks.BOOKMARK_TAG_NAME);
return skCase.getBlackboardArtifacts(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TAG_NAME, Tags.BOOKMARK_TAG_NAME);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Failed to get list of artifacts from the case.");
}
Core/src/org/sleuthkit/autopsy/directorytree/TagFileAction.java
+7 -7
View File
@@ -30,7 +30,6 @@ import javax.swing.JMenuItem;
import org.openide.nodes.Node;
import org.openide.util.actions.Presenter;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.Bookmarks;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.datamodel.Tags;
import org.sleuthkit.autopsy.directorytree.TagDialog.TagDialogResult;
@@ -71,13 +70,13 @@ public class TagFileAction extends AbstractAction implements Presenter.Popup {
@Override
public JMenuItem getPopupPresenter() {
JMenu result = new JMenu("Tag File");
JMenu result = new JMenu("Tag Source File");
JMenuItem contentItem = new JMenuItem("Bookmark File");
JMenuItem contentItem = new JMenuItem("Bookmark Source File");
contentItem.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
final TagDialog tagDialog = new TagDialog(TagDialog.Type.BOOKMARK, "Bookmark File", null, "Bookmark", false);
final TagDialog tagDialog = new TagDialog(TagDialog.Type.BOOKMARK, "Bookmark Source File", null, "Bookmark", false);
tagDialog.setVisible(true);
TagDialogResult inputResult = tagDialog.getResult();
if (inputResult.isAccept()) {
@@ -96,7 +95,7 @@ public class TagFileAction extends AbstractAction implements Presenter.Popup {
String newTagName = new CreateTagDialog(new JFrame(), true).display();
if (newTagName != null) {
//get comment
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag File", null, newTagName, false);
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag Source File", null, newTagName, false);
tagDialog.setVisible(true);
TagDialogResult inputResult = tagDialog.getResult();
if (inputResult.isAccept()) {
@@ -116,14 +115,15 @@ public class TagFileAction extends AbstractAction implements Presenter.Popup {
result.add(empty);
} else {
for (final String tagName : tagNames) {
if (tagName.equals(Bookmarks.BOOKMARK_TAG_NAME)) {
if (tagName.equals(Tags.BOOKMARK_TAG_NAME)) {
//skip showing bookmarks in tags menu, as they have a separate menu
continue;
}
JMenuItem tagItem = new JMenuItem(tagName);
tagItem.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag File", tagNames, tagName, true);
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag Source File", tagNames, tagName, true);
tagDialog.setVisible(true);
TagDialogResult inputResult = tagDialog.getResult();
if (inputResult.isAccept()) {
Core/src/org/sleuthkit/autopsy/directorytree/TagResultAction.java
+7 -7
View File
@@ -29,7 +29,6 @@ import javax.swing.JMenuItem;
import javax.swing.JOptionPane;
import org.openide.util.actions.Presenter;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.Bookmarks;
import org.sleuthkit.autopsy.datamodel.Tags;
import org.sleuthkit.datamodel.BlackboardArtifact;
@@ -58,13 +57,13 @@ public class TagResultAction extends AbstractAction implements Presenter.Popup {
@Override
public JMenuItem getPopupPresenter() {
JMenu result = new JMenu("Tag Result");
JMenu result = new JMenu("Tag Source Result");
JMenuItem contentItem = new JMenuItem("Bookmark Result");
JMenuItem contentItem = new JMenuItem("Bookmark Source Result");
contentItem.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
final TagDialog tagDialog = new TagDialog(TagDialog.Type.BOOKMARK, "Bookmark Result", null, "Bookmark", false);
final TagDialog tagDialog = new TagDialog(TagDialog.Type.BOOKMARK, "Bookmark Source Result", null, "Bookmark", false);
tagDialog.setVisible(true);
TagDialog.TagDialogResult inputResult = tagDialog.getResult();
if (inputResult.isAccept()) {
@@ -83,7 +82,7 @@ public class TagResultAction extends AbstractAction implements Presenter.Popup {
String newTagName = new CreateTagDialog(new JFrame(), true).display();
if (newTagName != null) {
//get comment and create tag
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag Result", null, newTagName, false);
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag Source Result", null, newTagName, false);
tagDialog.setVisible(true);
TagDialog.TagDialogResult inputResult = tagDialog.getResult();
if (inputResult.isAccept()) {
@@ -104,14 +103,15 @@ public class TagResultAction extends AbstractAction implements Presenter.Popup {
result.add(empty);
} else {
for (final String tagName : tagNames) {
if (tagName.equals(Bookmarks.BOOKMARK_TAG_NAME)) {
if (tagName.equals(Tags.BOOKMARK_TAG_NAME)) {
//skip showing bookmarks in tags menu, as they have a separate menu
continue;
}
JMenuItem tagItem = new JMenuItem(tagName);
tagItem.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag Result", tagNames, tagName, true);
final TagDialog tagDialog = new TagDialog(TagDialog.Type.TAG, "Tag Source Result", tagNames, tagName, true);
tagDialog.setVisible(true);
TagDialog.TagDialogResult inputResult = tagDialog.getResult();
if (inputResult.isAccept()) {
KeywordSearch/nbproject/project.xml
+2 -3
View File
@@ -100,8 +100,7 @@
</run-dependency>
</dependency>
</module-dependencies>
<friend-packages>
<friend>org.sleuthkit.autopsy.thunderbirdparser</friend>
<public-packages>
<package>org.apache.commons.lang</package>
<package>org.apache.commons.lang.builder</package>
<package>org.apache.commons.lang.enums</package>
@@ -166,7 +165,7 @@
<package>org.apache.tika.sax.xpath</package>
<package>org.apache.tika.utils</package>
<package>org.sleuthkit.autopsy.keywordsearch</package>
</friend-packages>
</public-packages>
<class-path-extension>
<runtime-relative-path>ext/metadata-extractor-2.4.0-beta-1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/metadata-extractor-2.4.0-beta-1.jar</binary-origin>
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java
+38 -35
View File
@@ -31,13 +31,9 @@ import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.S
import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Takes an AbstractFile, extract strings, converts into chunks (associated with the original
* source file) up to 1MB then and indexes chunks as text with Solr
* Takes an AbstractFile, extract strings, converts into chunks (associated with
* the original source file) up to 1MB then and indexes chunks as text with Solr
*/
class AbstractFileStringExtract implements AbstractFileExtract {
@@ -52,19 +48,23 @@ class AbstractFileStringExtract implements AbstractFileExtract {
//private static final int BOM_LEN = 3;
private static final int BOM_LEN = 0; //disabled prepending of BOM
private static final Charset INDEX_CHARSET = Server.DEFAULT_INDEXED_TEXT_CHARSET;
private static final SCRIPT DEFAULT_SCRIPT = SCRIPT.LATIN_2;
private final List<SCRIPT> extractScripts = new ArrayList<SCRIPT>();
private Map<String,String> extractOptions = new HashMap<String,String>();
private Map<String, String> extractOptions = new HashMap<String, String>();
//string extractor extracts from all other than archives
//TODO use content type detection mechanism
static final String[] UNSUPPORTED_EXTENSIONS = {
//Archives
//Note: archive unpacker module will process these instead
"tar", "jar", "zip", "7z", "gzip", "bzip", "bzip2", "gz", "tgz", "cab", "rar", "arj", "dmg", "iso"};
//disabled prepending of BOM
//static {
//prepend UTF-8 BOM to start of the buffer
//STRING_CHUNK_BUF[0] = (byte) 0xEF;
//STRING_CHUNK_BUF[1] = (byte) 0xBB;
//STRING_CHUNK_BUF[2] = (byte) 0xBF;
//prepend UTF-8 BOM to start of the buffer
//STRING_CHUNK_BUF[0] = (byte) 0xEF;
//STRING_CHUNK_BUF[1] = (byte) 0xBB;
//STRING_CHUNK_BUF[2] = (byte) 0xBF;
//}
public AbstractFileStringExtract() {
this.module = KeywordSearchIngestModule.getDefault();
this.ingester = Server.getIngester();
@@ -77,12 +77,11 @@ class AbstractFileStringExtract implements AbstractFileExtract {
this.extractScripts.addAll(extractScripts);
return true;
}
@Override
public List<SCRIPT> getScripts() {
return new ArrayList<SCRIPT>(this.extractScripts);
}
@Override
public int getNumChunks() {
@@ -103,42 +102,40 @@ class AbstractFileStringExtract implements AbstractFileExtract {
public void setOptions(Map<String, String> options) {
this.extractOptions = options;
}
@Override
public boolean index(AbstractFile sourceFile) throws IngesterException {
this.sourceFile = sourceFile;
this.numChunks = 0; //unknown until indexing is done
boolean success = false;
final boolean extractUTF8 =
final boolean extractUTF8 =
Boolean.parseBoolean(extractOptions.get(AbstractFileExtract.ExtractOptions.EXTRACT_UTF8.toString()));
final boolean extractUTF16 =
final boolean extractUTF16 =
Boolean.parseBoolean(extractOptions.get(AbstractFileExtract.ExtractOptions.EXTRACT_UTF16.toString()));
if (extractUTF8 == false && extractUTF16 == false) {
//nothing to do
return true;
}
InputStream stringStream = null;
//check which extract stream to use
if (extractScripts.size() == 1 && extractScripts.get(0).equals(SCRIPT.LATIN_1) ) {
if (extractScripts.size() == 1 && extractScripts.get(0).equals(SCRIPT.LATIN_1)) {
//optimal for english, english only
stringStream = new AbstractFileStringStream(sourceFile, INDEX_CHARSET);
}
else {
} else {
stringStream = new AbstractFileStringIntStream(
sourceFile, extractScripts, extractUTF8, extractUTF16, INDEX_CHARSET);
sourceFile, extractScripts, extractUTF8, extractUTF16, INDEX_CHARSET);
}
try {
success = true;
//break input stream into chunks
long readSize = 0;
while ((readSize = stringStream.read(STRING_CHUNK_BUF, BOM_LEN, (int) MAX_STRING_CHUNK_SIZE - BOM_LEN)) != -1) {
//FileOutputStream debug = new FileOutputStream("c:\\temp\\" + sourceFile.getName() + Integer.toString(this.numChunks+1));
@@ -183,16 +180,22 @@ class AbstractFileStringExtract implements AbstractFileExtract {
@Override
public boolean isContentTypeSpecific() {
return false;
return true;
}
@Override
public boolean isSupported(AbstractFile file) {
String fileNameLower = file.getName().toLowerCase();
int dotI = fileNameLower.lastIndexOf(".");
if (dotI == -1 || dotI == fileNameLower.length() - 1) {
return true; //no extension
}
final String extension = fileNameLower.substring(dotI + 1);
for (int i = 0; i < UNSUPPORTED_EXTENSIONS.length; ++i) {
if (extension.equals(UNSUPPORTED_EXTENSIONS[i])) {
return false;
}
}
return true;
}
}
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java
+18 -17
View File
@@ -66,23 +66,21 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
private int numChunks = 0;
//private static final String UTF16BOM = "\uFEFF"; disabled prepending of BOM
private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor();
// TODO: use a more robust method than checking file extension
// TODO: use type detection mechanism instead, and maintain supported MimeTypes, not extensions
// supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika
static final String[] SUPPORTED_EXTENSIONS = {
//Archives (TODO remove once we have extraction module)
"tar", "jar", "zip", "gzip", "bzip2", "gz", "tgz",
//MS Office
"doc", "dot", "docx", "docm", "dotx", "dotm",
"xls", "xlw", "xlt", "xlsx", "xlsm", "xltx", "xltm",
"ppt", "pps", "pot", "pptx", "pptm", "potx", "potm",
"xls", "xlw", "xlt", "xlsx", "xlsm", "xltx", "xltm",
"ppt", "pps", "pot", "pptx", "pptm", "potx", "potm",
//Open Office
"odf", "odt", "ott", "ods", "ots", "odp", "otp",
"sxw", "stw", "sxc", "stc", "sxi", "sxi",
"sdw", "sdc", "vor", "sgl",
"odf", "odt", "ott", "ods", "ots", "odp", "otp",
"sxw", "stw", "sxc", "stc", "sxi", "sxi",
"sdw", "sdc", "vor", "sgl",
//rich text, pdf
"rtf", "pdf",
"rtf", "pdf",
//html (other extractors take priority)
"html", "htm", "xhtml",
"html", "htm", "xhtml",
//text
"txt", "log", "manifest",
//images, media, other
@@ -92,7 +90,7 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
AbstractFileTikaTextExtract() {
this.module = KeywordSearchIngestModule.getDefault();
ingester = Server.getIngester();
}
@Override
@@ -104,15 +102,14 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
public List<StringExtract.StringExtractUnicodeTable.SCRIPT> getScripts() {
return null;
}
@Override
@Override
public Map<String, String> getOptions() {
return null;
}
@Override
public void setOptions(Map<String, String> options) {
}
@Override
@@ -283,8 +280,13 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
@Override
public boolean isSupported(AbstractFile file) {
String fileNameLower = file.getName().toLowerCase();
int dotI = fileNameLower.lastIndexOf(".");
if (dotI == -1 || dotI == fileNameLower.length() - 1) {
return false; //no extension
}
final String extension = fileNameLower.substring(dotI + 1);
for (int i = 0; i < SUPPORTED_EXTENSIONS.length; ++i) {
if (fileNameLower.endsWith(SUPPORTED_EXTENSIONS[i])) {
if (extension.equals(SUPPORTED_EXTENSIONS[i])) {
return true;
}
}
@@ -320,8 +322,7 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
KeywordSearch.getTikaLogger().log(Level.WARNING, "Unable to Tika parse the content" + sourceFile.getId() + ": " + sourceFile.getName(), ex);
tika = null;
reader = null;
}
catch (Exception ex) {
} catch (Exception ex) {
KeywordSearch.getTikaLogger().log(Level.WARNING, "Unable to Tika parse the content" + sourceFile.getId() + ": " + sourceFile.getName(), ex);
tika = null;
reader = null;
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
+3 -2
View File
@@ -696,7 +696,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
private boolean extractIndex(AbstractFile aFile, boolean stringsOnly) throws IngesterException {
AbstractFileExtract fileExtract = null;
if (stringsOnly) {
if (stringsOnly && stringExtractor.isSupported(aFile)) {
fileExtract = stringExtractor;
} else {
//go over available text extractors and pick the first one (most specific one)
@@ -709,7 +709,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
}
if (fileExtract == null) {
throw new IngesterException("No supported file extractor found for file: " + aFile.getId() + " " + aFile.getName());
logger.log(Level.INFO, "No supported file extractor found for file: " + aFile.getId() + " " + aFile.getName());
return false;
}
//logger.log(Level.INFO, "Extractor: " + fileExtract + ", file: " + aFile.getName());
build-windows.xml
+2 -1
View File
@@ -160,8 +160,9 @@
</target>
<target name="build-installer-windows">
<antcall target="run-advanced-installer" />
<antcall target="run-advanced-installer" />
<!--<delete dir="${nbdist.dir}/${app.name}-installer"/>-->
<delete dir="${nbdist.dir}/installer_${app.name}-cache"/>
<move file="${nbdist.dir}/installer_${app.name}-SetupFiles/installer_${app.name}.msi" tofile="${nbdist.dir}/installer_${app.name}-${app.version}.msi" />
</target>
</project>