From e30843c7c6bcc2d9bc48ff9deb39d75256ad5eff Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 10:15:45 -0500 Subject: [PATCH 01/12] Implemented record parsing and testing for call logs, web bookmarks and contacts --- .../xry/AbstractSingleKeyValueParser.java | 158 +++++++++++++++++ .../xry/XRYCallsFileParser.java | 165 ++++++++++++++++++ .../xry/XRYContactsFileParser.java | 76 ++++++++ .../xry/XRYFileParser.java | 46 +++++ .../xry/XRYFileParserFactory.java | 80 +++++++++ .../xry/XRYFileReader.java | 74 ++++++-- .../datasourceprocessors/xry/XRYFolder.java | 2 +- .../xry/XRYWebBookmarksFileParser.java | 68 ++++++++ 8 files changed, 658 insertions(+), 11 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParser.java create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java new file mode 100755 index 0000000000..0ed74bdbef --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -0,0 +1,158 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Template parse method for reports that make blackboard attributes from a + * single key value pairs. + * + * This parse implementation will create 1 artifact per XRY entity. + */ +abstract class AbstractSingleKeyValueParser implements XRYFileParser { + + private static final Logger logger = Logger.getLogger(AbstractSingleKeyValueParser.class.getName()); + + private static final char KEY_VALUE_DELIMITER = ':'; + + @Override + public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { + Path reportPath = reader.getReportPath(); + logger.log(Level.INFO, String.format("INFO: Processing report at [ %s ]", reportPath.toString())); + + while (reader.hasNextEntity()) { + String xryEntity = reader.nextEntity(); + String[] xryLines = xryEntity.split("\n"); + + List attributes = new ArrayList<>(); + + if (xryLines.length > 0) { + logger.log(Level.INFO, String.format("INFO: Processing [ %s ]", xryLines[0])); + } + + String namespace = ""; + for (int i = 1; i < xryLines.length; i++) { + String xryLine = xryLines[i]; + + if (isNamespace(xryLine)) { + logger.log(Level.INFO, String.format("INFO: Detected XRY " + + "namespace keyword [ %s ]. Applying to all key value pairs following it.", xryLine)); + namespace = xryLine.trim(); + continue; + } + + //Find the XRY key on this line. Assume key is the value between + //the start of the line and the first delimiter. + int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); + if (keyDelimiter == -1) { + logger.log(Level.SEVERE, String.format("Expected a key value " + + "pair on this line (in brackets) [ %s ], but one was not detected." + + " Here is the previous line (in brackets) [ %s ]. What does this key mean?", xryLine, xryLines[i - 1])); + continue; + } + String key = xryLine.substring(0, keyDelimiter).trim(); + String value = xryLine.substring(keyDelimiter + 1).trim(); + + if (!isKey(key)) { + logger.log(Level.SEVERE, String.format("The following key, " + + "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..." + + " Here is the previous line [ %s ] for context. What is it?", key, value, xryLines[i - 1])); + continue; + } + + if (value.isEmpty()) { + logger.log(Level.SEVERE, String.format("The following key " + + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding..." + + " Here is the previous line for context [ %s ]. What does this mean?", key, xryLines[i - 1])); + continue; + } + + BlackboardAttribute attribute = makeAttribute(namespace, key, value); + //Returning null is temporary solution until we map out how we will deal with + //attributes we are currently ignoring. + if (attribute != null) { + attributes.add(makeAttribute(namespace, key, value)); + } + } + + if (attributes.size() > 0) { + makeArtifact(attributes, parent); + } + } + } + + /** + * Determines if the key candidate is a known key. A key candidate is a + * string literal that begins a line and is terminated by a semi-colon. + * + * Ex: + * + * Call Type : Missed + * + * "Call Type" would be the key candidate that was extracted. + * + * @param key Key to test. These keys are trimmed of whitespace only. + * @return Indication if this key can be processed. + */ + abstract boolean isKey(String key); + + /** + * Determines if the namespace candidate is a known namespace. A namespace + * candidate is a string literal that makes up an entire line. + * + * Ex: + * + * To Tel : +1245325 + * + * "To" would be the candidate namespace that was extracted. + * + * @param nameSpace Namespace to test. Namespaces are trimmed of whitespace + * only. + * @return Indication if this namespace can be processed. + */ + abstract boolean isNamespace(String nameSpace); + + /** + * Creates an attribute from the extracted key value pair. + * + * @param nameSpace The namespace of this key value pair. + * It will have been verified with isNamespace, otherwise it will be empty. + * @param key The key that was verified with isKey. + * @param value The value associated with that key. + * @return + */ + abstract BlackboardAttribute makeAttribute(String nameSpace, String key, String value); + + /** + * Makes an artifact from the parsed attributes. + * + * @return + */ + abstract void makeArtifact(List attributes, Content parent) throws TskCoreException; + +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java new file mode 100755 index 0000000000..05934a4904 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -0,0 +1,165 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Set; +import java.util.HashSet; +import java.util.List; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Parses XRY Calls files and creates artifacts. + */ +final class XRYCallsFileParser extends AbstractSingleKeyValueParser { + + private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName()); + + //Human readable name of this parser. + private static final String PARSER_NAME = "XRY Calls"; + + private static final DateTimeFormatter DATE_TIME_PARSER + = DateTimeFormatter.ofPattern("M/d/y h:m:s [a][ z]"); + + private static final String INCOMING = "Incoming"; + + //All known XRY keys for call reports. + private static final Set XRY_KEYS = new HashSet() { + { + add("tel"); + add("number"); + add("call type"); + add("name (matched)"); + add("time"); + add("duration"); + add("storage"); + add("index"); + } + }; + + //All known XRY namespaces for call reports. + private static final Set XRY_NAMESPACES = new HashSet() { + { + add("to"); + add("from"); + } + }; + + @Override + boolean isKey(String key) { + String normalizedKey = key.toLowerCase(); + return XRY_KEYS.contains(normalizedKey); + } + + @Override + boolean isNamespace(String nameSpace) { + String normalizedNamespace = nameSpace.toLowerCase(); + return XRY_NAMESPACES.contains(normalizedNamespace); + } + + @Override + BlackboardAttribute makeAttribute(String nameSpace, String key, String value) { + String normalizedKey = key.toLowerCase(); + String normalizedNamespace = nameSpace.toLowerCase(); + + switch (normalizedKey) { + case "time": + //Tranform value to epoch ms + String dateTime = removeDateTimeLocale(value); + String normalizedDateTime = dateTime.trim(); + long dateTimeInEpoch = calculateMsSinceEpoch(normalizedDateTime); + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeInEpoch); + case "duration": + //Ignore for now. + return null; + case "storage": + //Ignore for now. + return null; + case "index": + //Ignore for now. + return null; + case "tel": + //Apply the namespace + switch (normalizedNamespace) { + case "from": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, PARSER_NAME, value); + default: + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, PARSER_NAME, value); + } + case "call type": + String normalizedValue = value.toLowerCase(); + switch (normalizedValue) { + case "missed": + case "received": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, PARSER_NAME, INCOMING); + case "dialed": + return null; + case "last dialed": + return null; + default: + logger.log(Level.SEVERE, String.format("Call type (in brackets) [ %s ] not recognized.", value)); + return null; + } + case "number": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, value); + case "name (matched)": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, PARSER_NAME, value); + default: + throw new IllegalArgumentException(String.format("key [ %s ] was not recognized.", key)); + } + } + + @Override + void makeArtifact(List attributes, Content parent) throws TskCoreException { + //BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); + //artifact.addAttributes(attributes); + } + + /** + * Removes the locale from the date time value. + * + * @param dateTime + * @return + */ + private String removeDateTimeLocale(String dateTime) { + int index = dateTime.indexOf('('); + if (index == -1) { + return dateTime; + } + + return dateTime.substring(0, index); + } + + /** + * + * @param dateTime + * @return + */ + private long calculateMsSinceEpoch(String dateTime) { + LocalDateTime localDateTime = LocalDateTime.parse(dateTime, DATE_TIME_PARSER); + //Assume dates have no offset. + return localDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java new file mode 100755 index 0000000000..233fd3da16 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -0,0 +1,76 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Parses XRY Contacts-Contacts files and creates artifacts. + */ +final class XRYContactsFileParser extends AbstractSingleKeyValueParser { + + //Human readable name of this parser. + private static final String PARSER_NAME = "XRY Contacts"; + + //All of the known XRY keys for contacts. + private static final Set XRY_KEYS = new HashSet() {{ + add("name"); + add("tel"); + add("storage"); + }}; + + @Override + boolean isKey(String key) { + String normalizedKey = key.toLowerCase(); + return XRY_KEYS.contains(normalizedKey); + } + + @Override + boolean isNamespace(String nameSpace) { + //No namespaces are currently known for this report type. + return false; + } + + @Override + BlackboardAttribute makeAttribute(String nameSpace, String key, String value) { + String normalizedKey = key.toLowerCase(); + switch(normalizedKey) { + case "name": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, PARSER_NAME, value); + case "tel": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, value); + case "storage": + //Ignore for now. + return null; + default: + throw new IllegalArgumentException(String.format("Key [ %s ] was not recognized", key)); + } + } + + @Override + void makeArtifact(List attributes, Content parent) throws TskCoreException { + //BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); + //artifact.addAttributes(attributes); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParser.java new file mode 100755 index 0000000000..1787641e78 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParser.java @@ -0,0 +1,46 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.io.IOException; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Interface for XRY file parsing. + */ +interface XRYFileParser { + + /** + * Parses XRY entities and creates artifacts from the interpreted content. + * + * See XRYFileReader for more information on XRY entities. It is expected + * that implementations will create artifacts on the supplied Content + * object. + * + * @param reader Produces XRY entities from a given XRY file. + * @param parent Content object that will act as the source of the + * artifacts. + * @throws IOException If an I/O error occurs during reading. + * @throws TskCoreException If an error occurs during artifact creation. + */ + void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException; + +} + diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java new file mode 100755 index 0000000000..8dd64996fa --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java @@ -0,0 +1,80 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +/** + * Instantiates XRYFileParsers by report type. + */ +final class XRYFileParserFactory { + + /** + * Creates the correct implementation of a XRYFileParser for the specified + * report type. + * + * It is assumed that the report type is supported, which means the client + * needs to have tested with supports beforehand. Otherwise, an + * IllegalArgumentException is thrown. + * + * @param reportType A supported XRY report type. + * @return A XRYFileParser with defined behavior for the report type. + * @throws IllegalArgumentException if the report type is not supported or + * is null. This is a misuse of the API. It is assumed that the report type + * has been tested with the supports method. + */ + public static XRYFileParser get(String reportType) { + if (reportType == null) { + throw new IllegalArgumentException("Report type cannot be null"); + } + + switch (reportType.toLowerCase()) { + case "calls": + return new XRYCallsFileParser(); + case "contacts/contacts": + return new XRYContactsFileParser(); + case "device/general information": + return new XRYDeviceGenInfoFileParser(); + case "messages/sms": + return new XRYMessagesFileParser(); + case "web/bookmarks": + return new XRYWebBookmarksFileParser(); + default: + throw new IllegalArgumentException(reportType + " not recognized."); + } + } + + /** + * Tests if a XRYFileParser implementation exists for the report type. + * + * @param reportType Report type to test. + * @return Indication if the report type can be parsed. + */ + public static boolean supports(String reportType) { + try { + //Attempt a get. + get(reportType); + return true; + } catch (IllegalArgumentException ex) { + return false; + } + } + + //Prevent direct instantiation + private XRYFileParserFactory() { + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java index ff854a16ea..32a0b7eb33 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java @@ -45,36 +45,43 @@ import org.apache.commons.io.FilenameUtils; * From * Tel: 12345678 */ -public final class XRYFileReader implements AutoCloseable { +final class XRYFileReader implements AutoCloseable { private static final Logger logger = Logger.getLogger(XRYFileReader.class.getName()); //Assume UTF_16LE private static final Charset CHARSET = StandardCharsets.UTF_16LE; - //Assume TXT extension - private static final String EXTENSION = "txt"; - - //Assume 0xFFFE is the BOM - private static final int[] BOM = {0xFF, 0xFE}; - //Assume all XRY reports have the type on the 3rd line. private static final int LINE_WITH_REPORT_TYPE = 3; //Assume all headers are 5 lines in length. private static final int HEADER_LENGTH_IN_LINES = 5; + //Assume TXT extension + private static final String EXTENSION = "txt"; + + //Assume 0xFFFE is the BOM + private static final int[] BOM = {0xFF, 0xFE}; + + //Entity to be consumed during file iteration. + private final StringBuilder xryEntity; + //Underlying reader for the xry file. private final BufferedReader reader; - private final StringBuilder xryEntity; + //Reference to the original xry file. + private final Path xryFilePath; /** * Creates an XRYFileReader. As part of construction, the XRY file is opened * and the reader is advanced past the header. This leaves the reader * positioned at the start of the first XRY entity. * - * The file is assumed to be encoded in UTF-16LE. + * The file is assumed to be encoded in UTF-16LE and is NOT verified to be + * an XRY file before reading. It is expected that the isXRYFile function + * has been called on the path beforehand. Otherwise, the behavior is + * undefined. * * @param xryFile XRY file to read. It is assumed that the caller has read * access to the path. @@ -82,6 +89,7 @@ public final class XRYFileReader implements AutoCloseable { */ public XRYFileReader(Path xryFile) throws IOException { reader = Files.newBufferedReader(xryFile, CHARSET); + xryFilePath = xryFile; //Advance the reader to the start of the first XRY entity. for (int i = 0; i < HEADER_LENGTH_IN_LINES; i++) { @@ -91,6 +99,35 @@ public final class XRYFileReader implements AutoCloseable { xryEntity = new StringBuilder(); } + /** + * Extracts the report type from the XRY file. + * + * @return The XRY report type + * @throws IOException if an I/O error occurs. + * @throws IllegalArgumentExcepton If the XRY file does not have a report + * type. This is a misuse of the API. The validity of the Path should have + * been checked with isXRYFile before creating an XRYFileReader. + */ + public String getReportType() throws IOException { + Optional reportType = getType(xryFilePath); + if (reportType.isPresent()) { + return reportType.get(); + } + + throw new IllegalArgumentException(xryFilePath.toString() + " does not " + + "have a report type."); + } + + /** + * Returns the raw path of the XRY report file. + * + * @return + * @throws IOException + */ + public Path getReportPath() throws IOException { + return xryFilePath; + } + /** * Advances the reader until a valid XRY entity is detected or EOF is * reached. @@ -113,7 +150,7 @@ public final class XRYFileReader implements AutoCloseable { return true; } } else { - xryEntity.append(line).append("\n"); + xryEntity.append(line).append('\n'); } } @@ -138,6 +175,23 @@ public final class XRYFileReader implements AutoCloseable { throw new NoSuchElementException(); } } + + /** + * Peek at the next XRY entity without consuming it. + * If there are not more XRY entities left, an exception is thrown. + * + * @return A non-empty XRY entity. + * @throws IOException + * @throws NoSuchElementException if there are no more XRY entities to + * read. + */ + public String peek() throws IOException { + if(hasNextEntity()) { + return xryEntity.toString(); + } else { + throw new NoSuchElementException(); + } + } /** * Closes any file handles this reader may have open. diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFolder.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFolder.java index b9b999f270..bac2bc2364 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFolder.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFolder.java @@ -30,7 +30,7 @@ import java.util.stream.Stream; /** * Extracts XRY files and (optionally) non-XRY files from a XRY (Report) folder. */ -public final class XRYFolder { +final class XRYFolder { //Depth that will contain XRY files. All XRY files will be immediate //children of their parent folder. diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java new file mode 100755 index 0000000000..9cc6a4d745 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -0,0 +1,68 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.util.Map; +import java.util.HashMap; +import java.util.List; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Parses XRY Web-Bookmark files and creates artifacts. + */ +final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { + + //Human readable name of this parser. + private static final String PARSER_NAME = "XRY Web Bookmarks"; + + //All known XRY keys for web bookmarks. + private static final Map KEY_TO_TYPE + = new HashMap() { + { + put("web address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL); + put("domain", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN); + } + }; + + @Override + boolean isKey(String key) { + String normalizedKey = key.toLowerCase(); + return KEY_TO_TYPE.containsKey(normalizedKey); + } + + @Override + boolean isNamespace(String nameSpace) { + //No known namespaces for web reports. + return false; + } + + @Override + BlackboardAttribute makeAttribute(String nameSpace, String key, String value) { + String normalizedKey = key.toLowerCase(); + return new BlackboardAttribute(KEY_TO_TYPE.get(normalizedKey), PARSER_NAME, value); + } + + @Override + void makeArtifact(List attributes, Content parent) throws TskCoreException { + //BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.WEB_BOOKMARK); + //artifact.addAttributes(attributes); + } +} From e67da6aec6663dfa537c4ffff53ccf918d138db8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 10:16:50 -0500 Subject: [PATCH 02/12] Uncommented to artifact creation code --- .../autopsy/datasourceprocessors/xry/XRYCallsFileParser.java | 4 ++-- .../datasourceprocessors/xry/XRYContactsFileParser.java | 4 ++-- .../datasourceprocessors/xry/XRYWebBookmarksFileParser.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 05934a4904..f422cd1ead 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -133,8 +133,8 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { @Override void makeArtifact(List attributes, Content parent) throws TskCoreException { - //BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); - //artifact.addAttributes(attributes); + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); + artifact.addAttributes(attributes); } /** diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index 233fd3da16..ac74ae9ada 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -70,7 +70,7 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { @Override void makeArtifact(List attributes, Content parent) throws TskCoreException { - //BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); - //artifact.addAttributes(attributes); + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); + artifact.addAttributes(attributes); } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index 9cc6a4d745..b9f2528f33 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -62,7 +62,7 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { @Override void makeArtifact(List attributes, Content parent) throws TskCoreException { - //BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.WEB_BOOKMARK); - //artifact.addAttributes(attributes); + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.WEB_BOOKMARK); + artifact.addAttributes(attributes); } } From c1117164037c4008c1a26d77ccb933468a5eb3ed Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 10:26:28 -0500 Subject: [PATCH 03/12] Tied together any loose ends. Added imports and fixed comments --- .../xry/AbstractSingleKeyValueParser.java | 14 +++++++++----- .../xry/XRYCallsFileParser.java | 19 ++++++++++++------- .../xry/XRYContactsFileParser.java | 1 + .../xry/XRYWebBookmarksFileParser.java | 1 + 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java index 0ed74bdbef..317ca17299 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -30,7 +30,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Template parse method for reports that make blackboard attributes from a - * single key value pairs. + * single key value pair. * * This parse implementation will create 1 artifact per XRY entity. */ @@ -59,6 +59,8 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { for (int i = 1; i < xryLines.length; i++) { String xryLine = xryLines[i]; + //Check if the line is a namespace, which gives context to the keys + //that follow. if (isNamespace(xryLine)) { logger.log(Level.INFO, String.format("INFO: Detected XRY " + "namespace keyword [ %s ]. Applying to all key value pairs following it.", xryLine)); @@ -81,7 +83,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { if (!isKey(key)) { logger.log(Level.SEVERE, String.format("The following key, " + "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..." - + " Here is the previous line [ %s ] for context. What is it?", key, value, xryLines[i - 1])); + + " Here is the previous line [ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); continue; } @@ -93,13 +95,14 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { } BlackboardAttribute attribute = makeAttribute(namespace, key, value); - //Returning null is temporary solution until we map out how we will deal with - //attributes we are currently ignoring. + //Temporarily allowing null to be valid return type until a decision + //is made about how to handle keys we are choosing to ignore. if (attribute != null) { attributes.add(makeAttribute(namespace, key, value)); } } + //Only create artifacts with non-empty attributes. if (attributes.size() > 0) { makeArtifact(attributes, parent); } @@ -127,7 +130,8 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { * * Ex: * - * To Tel : +1245325 + * To + * Tel : +1245325 * * "To" would be the candidate namespace that was extracted. * diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index f422cd1ead..e417703606 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -26,6 +26,7 @@ import java.util.HashSet; import java.util.List; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; @@ -34,7 +35,7 @@ import org.sleuthkit.datamodel.TskCoreException; * Parses XRY Calls files and creates artifacts. */ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { - + private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName()); //Human readable name of this parser. @@ -42,7 +43,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { private static final DateTimeFormatter DATE_TIME_PARSER = DateTimeFormatter.ofPattern("M/d/y h:m:s [a][ z]"); - + private static final String INCOMING = "Incoming"; //All known XRY keys for call reports. @@ -113,7 +114,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { switch (normalizedValue) { case "missed": case "received": - return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, PARSER_NAME, INCOMING); + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, PARSER_NAME, INCOMING); case "dialed": return null; case "last dialed": @@ -130,7 +131,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { throw new IllegalArgumentException(String.format("key [ %s ] was not recognized.", key)); } } - + @Override void makeArtifact(List attributes, Content parent) throws TskCoreException { BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); @@ -139,9 +140,11 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { /** * Removes the locale from the date time value. + * + * Locale in this case being (Device) or (Network). * - * @param dateTime - * @return + * @param dateTime XRY datetime value to be sanitized. + * @return A purer date time value. */ private String removeDateTimeLocale(String dateTime) { int index = dateTime.indexOf('('); @@ -153,7 +156,9 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } /** - * + * Parses the datatime value and calculates ms since epoch. It time zone is + * assumed to be UTC. + * * @param dateTime * @return */ diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index ac74ae9ada..ff47f33e95 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index b9f2528f33..e3ba5d6f23 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.HashMap; import java.util.List; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; From d39ed4d7323a65819fcd0f081f1c9c6f09f8917d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 10:45:25 -0500 Subject: [PATCH 04/12] Fixed type in artifact naem --- .../datasourceprocessors/xry/XRYWebBookmarksFileParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index e3ba5d6f23..3c6e881ada 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -63,7 +63,7 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { @Override void makeArtifact(List attributes, Content parent) throws TskCoreException { - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.WEB_BOOKMARK); + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK); artifact.addAttributes(attributes); } } From f181a1f862fafd51b9820bfe7d96f95033c84ac1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 10:55:51 -0500 Subject: [PATCH 05/12] Updated comments --- .../xry/AbstractSingleKeyValueParser.java | 4 +++- .../datasourceprocessors/xry/XRYCallsFileParser.java | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java index 317ca17299..d3a031a776 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -56,6 +56,8 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { } String namespace = ""; + //Process each line, searching for a key value pair or a namespace. + //If neither are found, an error message is logged. for (int i = 1; i < xryLines.length; i++) { String xryLine = xryLines[i]; @@ -74,7 +76,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { if (keyDelimiter == -1) { logger.log(Level.SEVERE, String.format("Expected a key value " + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Here is the previous line (in brackets) [ %s ]. What does this key mean?", xryLine, xryLines[i - 1])); + + " Here is the previous line (in brackets) [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); continue; } String key = xryLine.substring(0, keyDelimiter).trim(); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index e417703606..b4c556079c 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -140,7 +140,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { /** * Removes the locale from the date time value. - * + * * Locale in this case being (Device) or (Network). * * @param dateTime XRY datetime value to be sanitized. @@ -156,9 +156,9 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } /** - * Parses the datatime value and calculates ms since epoch. It time zone is + * Parses the datatime value and calculates ms since epoch. The time zone is * assumed to be UTC. - * + * * @param dateTime * @return */ From 36e9c7a5c63369e55df51069c2769c3a7cbf3b1e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 11:17:48 -0500 Subject: [PATCH 06/12] Addressed some of the codacy comments --- .../xry/AbstractSingleKeyValueParser.java | 2 +- .../datasourceprocessors/xry/XRYCallsFileParser.java | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java index d3a031a776..b5f4220f85 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -105,7 +105,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { } //Only create artifacts with non-empty attributes. - if (attributes.size() > 0) { + if (!attributes.isEmpty()) { makeArtifact(attributes, parent); } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index b4c556079c..297f3d06cd 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -103,11 +103,10 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { return null; case "tel": //Apply the namespace - switch (normalizedNamespace) { - case "from": - return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, PARSER_NAME, value); - default: - return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, PARSER_NAME, value); + if(normalizedNamespace.equals("from")) { + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, PARSER_NAME, value); + } else { + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, PARSER_NAME, value); } case "call type": String normalizedValue = value.toLowerCase(); From a2584ef060059021bb8ad66185e973fedee37544 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 11:20:37 -0500 Subject: [PATCH 07/12] Fixed typo in comment and added a line for clarity --- .../datasourceprocessors/xry/AbstractSingleKeyValueParser.java | 1 + .../autopsy/datasourceprocessors/xry/XRYCallsFileParser.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java index b5f4220f85..9ebf7ec32b 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -51,6 +51,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { List attributes = new ArrayList<>(); + //First line of the entity is the title. if (xryLines.length > 0) { logger.log(Level.INFO, String.format("INFO: Processing [ %s ]", xryLines[0])); } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 297f3d06cd..48aff2e762 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -155,7 +155,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } /** - * Parses the datatime value and calculates ms since epoch. The time zone is + * Parses the date time value and calculates ms since epoch. The time zone is * assumed to be UTC. * * @param dateTime From eb1a9cd279a295a2e0d5e77ef978425d8e199269 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 12:17:55 -0500 Subject: [PATCH 08/12] Implemented and tested the device gen info parser --- .../xry/XRYDeviceGenInfoFileParser.java | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java new file mode 100755 index 0000000000..937627d61d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -0,0 +1,189 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Parses XRY Device-General Information files and creates artifacts. + */ +final class XRYDeviceGenInfoFileParser implements XRYFileParser { + + private static final Logger logger = Logger.getLogger(XRYDeviceGenInfoFileParser.class.getName()); + + //Human readable name of this parser. + private static final String PARSER_NAME = "XRY Device General Info"; + private static final char KEY_VALUE_DELIMITER = ':'; + + //All known XRY keys for Device Gen Info reports. + private static final String ATTRIBUTE_KEY = "attribute"; + private static final String DATA_KEY = "data"; + + //All of the known XRY keys for device gen info. + private static final Map KEY_TO_TYPE + = new HashMap() { + { + put("device name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_NAME); + put("device family", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MODEL); + put("device type", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MAKE); + put("mobile id (imei)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI); + put("security code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD); + } + }; + + /** + * Device-General Information reports have 2 key value pairs for every + * attribute. The two only known keys are "Data" and "Attribute", where data + * is some generic information that the Attribute key describes. + * + * Example: + * + * Data: Nokia XYZ + * Attribute: Device Name + * + * This parse implementation assumes that the data field does not span + * multiple lines. If the data does span multiple lines, it will log an + * error describing an expectation for an "Attribute" key that is not found. + * + * @param reader The XRYFileReader that reads XRY entities from the + * Device-General Information report. + * @param parent The parent Content to create artifacts from. + * @throws IOException + * @throws TskCoreException + */ + @Override + public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { + Path reportPath = reader.getReportPath(); + logger.log(Level.INFO, String.format("Processing report at [ %s ]", reportPath.toString())); + + while (reader.hasNextEntity()) { + String xryEntity = reader.nextEntity(); + String[] xryLines = xryEntity.split("\n"); + + List attributes = new ArrayList<>(); + + //First line of the entity is the title. + if (xryLines.length > 0) { + logger.log(Level.INFO, String.format("Processing [ %s ]", xryLines[0])); + } + + for (int i = 1; i < xryLines.length; i++) { + String xryLine = xryLines[i]; + + //Expecting to see a "Data" key. + if (!hasDataKey(xryLine)) { + logger.log(Level.SEVERE, String.format("Expected a 'Data' key " + + "on this line (in brackets) [ %s ], but none was found. " + + "Discarding... Here is the previous line for context [ %s ]. " + + "What does this mean?", xryLine, xryLines[i - 1])); + continue; + } + + if (i + 1 == xryLines.length) { + logger.log(Level.SEVERE, String.format("Found a 'Data' key " + + "but no corresponding 'Attribute' key. Discarding... Here " + + "is the 'Data' line (in brackets) [ %s ]. Here is the previous " + + "line for context [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); + continue; + } + + int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); + String dataValue = xryLine.substring(dataKeyIndex + 1).trim(); + + String nextXryLine = xryLines[++i]; + + //Expecting to see an "Attribute" key + if (!hasAttributeKey(nextXryLine)) { + logger.log(Level.SEVERE, String.format("SEVERE: Expected an 'Attribute' " + + "key on this line (in brackets) [ %s ], but none was found. " + + "Discarding... Here is the previous line for context [ %s ]. " + + "What does this mean?", nextXryLine, xryLine)); + continue; + } + + int attributeKeyIndex = nextXryLine.indexOf(KEY_VALUE_DELIMITER); + String attributeValue = nextXryLine.substring(attributeKeyIndex + 1).trim(); + String normalizedAttributeValue = attributeValue.toLowerCase(); + + //Check if the attribute value is recognized. + if (KEY_TO_TYPE.containsKey(normalizedAttributeValue)) { + //All of the attribute types in the map expect a string. + attributes.add(new BlackboardAttribute(KEY_TO_TYPE.get(normalizedAttributeValue), PARSER_NAME, dataValue)); + } else { + logger.log(Level.SEVERE, String.format("Attribute type (in brackets) " + + "[ %s ] was not recognized. Discarding... Here is the " + + "previous line for context [ %s ]. What does this mean?", nextXryLine, xryLine)); + } + } + + if(!attributes.isEmpty()) { + //Build the artifact. + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO); + artifact.addAttributes(attributes); + } + } + } + + /** + * Determines if the XRY line has a data key on it. + * + * @param xryLine + * @return + */ + private boolean hasDataKey(String xryLine) { + int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); + //No key structure found. + if (dataKeyIndex == -1) { + return false; + } + + String normalizedDataKey = xryLine.substring(0, + dataKeyIndex).trim().toLowerCase(); + return normalizedDataKey.equals(DATA_KEY); + } + + /** + * Determines if the XRY line has an attribute key on it. + * + * @param xryLine + * @return + */ + private boolean hasAttributeKey(String xryLine) { + int attributeKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); + //No key structure found. + if (attributeKeyIndex == -1) { + return false; + } + + String normalizedDataKey = xryLine.substring(0, + attributeKeyIndex).trim().toLowerCase(); + return normalizedDataKey.equals(ATTRIBUTE_KEY); + } +} From 3c4aaa3f8dbe8d08ecd63d372b7e5629256036a2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 12:24:16 -0500 Subject: [PATCH 09/12] Updated comments --- .../xry/XRYDeviceGenInfoFileParser.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index 937627d61d..5b50e534a6 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -46,7 +46,9 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { private static final String ATTRIBUTE_KEY = "attribute"; private static final String DATA_KEY = "data"; - //All of the known XRY keys for device gen info. + //All of the known XRY Attribute values for device gen info. The value of the + //attribute keys are actionable for this parser. See parse header for more + //details. private static final Map KEY_TO_TYPE = new HashMap() { { @@ -75,8 +77,8 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { * @param reader The XRYFileReader that reads XRY entities from the * Device-General Information report. * @param parent The parent Content to create artifacts from. - * @throws IOException - * @throws TskCoreException + * @throws IOException If an I/O error is encountered during report reading + * @throws TskCoreException If an error during artifact creation is encountered. */ @Override public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { From 1144649e370de6696991d527ca40e2c12c0eb3f9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 15:52:38 -0500 Subject: [PATCH 10/12] Implemented and tested XRY messages parser, implemented code review feedback --- .../xry/AbstractSingleKeyValueParser.java | 16 +- .../xry/XRYCallsFileParser.java | 18 +- .../xry/XRYContactsFileParser.java | 3 - .../xry/XRYDeviceGenInfoFileParser.java | 14 +- .../xry/XRYFileParserFactory.java | 4 +- .../xry/XRYFileReader.java | 25 +- .../xry/XRYMessagesFileParser.java | 483 ++++++++++++++++++ .../xry/XRYWebBookmarksFileParser.java | 3 - 8 files changed, 524 insertions(+), 42 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java index 9ebf7ec32b..20b6b7c1bf 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -39,11 +39,13 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { private static final Logger logger = Logger.getLogger(AbstractSingleKeyValueParser.class.getName()); private static final char KEY_VALUE_DELIMITER = ':'; + + protected static final String PARSER_NAME = "XRY DSP"; @Override public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { Path reportPath = reader.getReportPath(); - logger.log(Level.INFO, String.format("INFO: Processing report at [ %s ]", reportPath.toString())); + logger.log(Level.INFO, String.format("XRY DSP: Processing report at [ %s ]", reportPath.toString())); while (reader.hasNextEntity()) { String xryEntity = reader.nextEntity(); @@ -53,7 +55,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { //First line of the entity is the title. if (xryLines.length > 0) { - logger.log(Level.INFO, String.format("INFO: Processing [ %s ]", xryLines[0])); + logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ]", xryLines[0])); } String namespace = ""; @@ -65,8 +67,6 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { //Check if the line is a namespace, which gives context to the keys //that follow. if (isNamespace(xryLine)) { - logger.log(Level.INFO, String.format("INFO: Detected XRY " - + "namespace keyword [ %s ]. Applying to all key value pairs following it.", xryLine)); namespace = xryLine.trim(); continue; } @@ -75,23 +75,23 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { //the start of the line and the first delimiter. int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); if (keyDelimiter == -1) { - logger.log(Level.SEVERE, String.format("Expected a key value " + logger.log(Level.SEVERE, String.format("XRY DSP: Expected a key value " + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Here is the previous line (in brackets) [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); + + " Here is the previous line [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); continue; } String key = xryLine.substring(0, keyDelimiter).trim(); String value = xryLine.substring(keyDelimiter + 1).trim(); if (!isKey(key)) { - logger.log(Level.SEVERE, String.format("The following key, " + logger.log(Level.SEVERE, String.format("XRY DSP: The following key, " + "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..." + " Here is the previous line [ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); continue; } if (value.isEmpty()) { - logger.log(Level.SEVERE, String.format("The following key " + logger.log(Level.SEVERE, String.format("XRY DSP: The following key " + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding..." + " Here is the previous line for context [ %s ]. What does this mean?", key, xryLines[i - 1])); continue; diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 48aff2e762..af59c6891d 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.Set; import java.util.HashSet; import java.util.List; @@ -38,9 +39,6 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName()); - //Human readable name of this parser. - private static final String PARSER_NAME = "XRY Calls"; - private static final DateTimeFormatter DATE_TIME_PARSER = DateTimeFormatter.ofPattern("M/d/y h:m:s [a][ z]"); @@ -88,10 +86,16 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { switch (normalizedKey) { case "time": //Tranform value to epoch ms - String dateTime = removeDateTimeLocale(value); - String normalizedDateTime = dateTime.trim(); - long dateTimeInEpoch = calculateMsSinceEpoch(normalizedDateTime); - return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeInEpoch); + try { + String dateTime = removeDateTimeLocale(value); + String normalizedDateTime = dateTime.trim(); + long dateTimeInEpoch = calculateMsSinceEpoch(normalizedDateTime); + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeInEpoch); + } catch (DateTimeParseException ex) { + logger.log(Level.SEVERE, String.format("XRY DSP: Assumption about the date time " + + "formatting of call logs is not right. Here is the value [ %s ]", value), ex); + return null; + } case "duration": //Ignore for now. return null; diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index ff47f33e95..ec8fd40187 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -31,9 +31,6 @@ import org.sleuthkit.datamodel.TskCoreException; */ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { - //Human readable name of this parser. - private static final String PARSER_NAME = "XRY Contacts"; - //All of the known XRY keys for contacts. private static final Set XRY_KEYS = new HashSet() {{ add("name"); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index 5b50e534a6..d610ffb89d 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -39,7 +39,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { private static final Logger logger = Logger.getLogger(XRYDeviceGenInfoFileParser.class.getName()); //Human readable name of this parser. - private static final String PARSER_NAME = "XRY Device General Info"; + private static final String PARSER_NAME = "XRY DSP"; private static final char KEY_VALUE_DELIMITER = ':'; //All known XRY keys for Device Gen Info reports. @@ -83,7 +83,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { @Override public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { Path reportPath = reader.getReportPath(); - logger.log(Level.INFO, String.format("Processing report at [ %s ]", reportPath.toString())); + logger.log(Level.INFO, String.format("XRY DSP: Processing report at [ %s ]", reportPath.toString())); while (reader.hasNextEntity()) { String xryEntity = reader.nextEntity(); @@ -93,7 +93,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //First line of the entity is the title. if (xryLines.length > 0) { - logger.log(Level.INFO, String.format("Processing [ %s ]", xryLines[0])); + logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ]", xryLines[0])); } for (int i = 1; i < xryLines.length; i++) { @@ -101,7 +101,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //Expecting to see a "Data" key. if (!hasDataKey(xryLine)) { - logger.log(Level.SEVERE, String.format("Expected a 'Data' key " + logger.log(Level.SEVERE, String.format("XRY DSP: Expected a 'Data' key " + "on this line (in brackets) [ %s ], but none was found. " + "Discarding... Here is the previous line for context [ %s ]. " + "What does this mean?", xryLine, xryLines[i - 1])); @@ -109,7 +109,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { } if (i + 1 == xryLines.length) { - logger.log(Level.SEVERE, String.format("Found a 'Data' key " + logger.log(Level.SEVERE, String.format("XRY DSP: Found a 'Data' key " + "but no corresponding 'Attribute' key. Discarding... Here " + "is the 'Data' line (in brackets) [ %s ]. Here is the previous " + "line for context [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); @@ -123,7 +123,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //Expecting to see an "Attribute" key if (!hasAttributeKey(nextXryLine)) { - logger.log(Level.SEVERE, String.format("SEVERE: Expected an 'Attribute' " + logger.log(Level.SEVERE, String.format("XRY DSP: Expected an 'Attribute' " + "key on this line (in brackets) [ %s ], but none was found. " + "Discarding... Here is the previous line for context [ %s ]. " + "What does this mean?", nextXryLine, xryLine)); @@ -139,7 +139,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //All of the attribute types in the map expect a string. attributes.add(new BlackboardAttribute(KEY_TO_TYPE.get(normalizedAttributeValue), PARSER_NAME, dataValue)); } else { - logger.log(Level.SEVERE, String.format("Attribute type (in brackets) " + logger.log(Level.SEVERE, String.format("XRY DSP: Attribute type (in brackets) " + "[ %s ] was not recognized. Discarding... Here is the " + "previous line for context [ %s ]. What does this mean?", nextXryLine, xryLine)); } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java index 8dd64996fa..d650510827 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java @@ -37,7 +37,7 @@ final class XRYFileParserFactory { * is null. This is a misuse of the API. It is assumed that the report type * has been tested with the supports method. */ - public static XRYFileParser get(String reportType) { + static XRYFileParser get(String reportType) { if (reportType == null) { throw new IllegalArgumentException("Report type cannot be null"); } @@ -64,7 +64,7 @@ final class XRYFileParserFactory { * @param reportType Report type to test. * @return Indication if the report type can be parsed. */ - public static boolean supports(String reportType) { + static boolean supports(String reportType) { try { //Attempt a get. get(reportType); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java index 32a0b7eb33..bc3c04259b 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileReader.java @@ -101,7 +101,7 @@ final class XRYFileReader implements AutoCloseable { /** * Extracts the report type from the XRY file. - * + * * @return The XRY report type * @throws IOException if an I/O error occurs. * @throws IllegalArgumentExcepton If the XRY file does not have a report @@ -117,12 +117,12 @@ final class XRYFileReader implements AutoCloseable { throw new IllegalArgumentException(xryFilePath.toString() + " does not " + "have a report type."); } - + /** * Returns the raw path of the XRY report file. - * + * * @return - * @throws IOException + * @throws IOException */ public Path getReportPath() throws IOException { return xryFilePath; @@ -160,6 +160,7 @@ final class XRYFileReader implements AutoCloseable { /** * Returns an XRY entity if there is one, otherwise an exception is thrown. + * Clients should test for another entity by calling hasNextEntity(). * * @return A non-empty XRY entity. * @throws IOException if an I/O error occurs. @@ -175,18 +176,18 @@ final class XRYFileReader implements AutoCloseable { throw new NoSuchElementException(); } } - + /** - * Peek at the next XRY entity without consuming it. - * If there are not more XRY entities left, an exception is thrown. - * + * Peek at the next XRY entity without consuming it. If there are not more + * XRY entities left, an exception is thrown. Clients should test for + * another entity by calling hasNextEntity(). + * * @return A non-empty XRY entity. - * @throws IOException - * @throws NoSuchElementException if there are no more XRY entities to - * read. + * @throws IOException if an I/O error occurs. + * @throws NoSuchElementException if there are no more XRY entities to peek. */ public String peek() throws IOException { - if(hasNextEntity()) { + if (hasNextEntity()) { return xryEntity.toString(); } else { throw new NoSuchElementException(); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java new file mode 100755 index 0000000000..c1a80f59ae --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -0,0 +1,483 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Parses Messages-SMS files and creates artifacts. + */ +final class XRYMessagesFileParser implements XRYFileParser { + + private static final Logger logger = Logger.getLogger( + XRYMessagesFileParser.class.getName()); + + private static final String PARSER_NAME = "XRY DSP"; + private static final char KEY_VALUE_DELIMITER = ':'; + private static final DateTimeFormatter DATE_TIME_PARSER + = DateTimeFormatter.ofPattern("M/d/y h:m:s [a][ z]"); + + //Meta keys. These describe how the XRY message entites are split + //up in the report file. + private static final String SEGMENT_COUNT = "segments"; + private static final String SEGMENT_NUMBER = "segment number"; + private static final String REFERENCE_NUMBER = "reference number"; + + //A more readable version of these values. Referring to if the user + //has read the message. + private static final int READ = 1; + private static final int UNREAD = 0; + + private static final String TEXT_KEY = "text"; + + //All known XRY keys for message reports. + private static final Set XRY_KEYS = new HashSet() { + { + add(TEXT_KEY); + add("direction"); + add("time"); + add("status"); + add("tel"); + add("storage"); + add("index"); + add("folder"); + add("service center"); + add("type"); + } + }; + + //All known XRY namespaces for message reports. + private static final Set XRY_NAMESPACES = new HashSet() { + { + add("to"); + add("from"); + add("participant"); + } + }; + + //All known meta keys. + private static final Set XRY_META_KEYS = new HashSet() { + { + add(REFERENCE_NUMBER); + add(SEGMENT_NUMBER); + add(SEGMENT_COUNT); + } + }; + + /** + * Message-SMS report artifacts can span multiple XRY entities and their + * attributes can span multiple lines. The "Text" key is the only known key + * value pair that can span multiple lines. Messages can be segmented, + * meaning that their "Text" content can appear in multiple XRY entities. + * Our goal for a segmented message is aggregate all of the text pieces and + * create 1 artifact. + * + * This parse implementation assumes that segments are contiguous and that + * they ascend incrementally. There are checks in place to verify this + * assumption are correct, otherwise an error will appear in the logs. + * + * @param reader The XRYFileReader that reads XRY entities from the + * Message-SMS report. + * @param parent The parent Content to create artifacts from. + * @throws IOException If an I/O error is encountered during report reading + * @throws TskCoreException If an error during artifact creation is + * encountered. + */ + @Override + public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { + Path reportPath = reader.getReportPath(); + logger.log(Level.INFO, String.format("XRY DSP: Processing report at [ %s ]", reportPath.toString())); + + //Keep track of the reference numbers that have been parsed. + Set referenceNumbersSeen = new HashSet<>(); + + while (reader.hasNextEntity()) { + String xryEntity = reader.nextEntity(); + String[] xryLines = xryEntity.split("\n"); + + //First line of the entity is the title. + if (xryLines.length > 0) { + logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ]", xryLines[0])); + } + + List attributes = new ArrayList<>(); + + String namespace = ""; + for (int i = 1; i < xryLines.length; i++) { + String xryLine = xryLines[i]; + String normalizedXryLine = xryLine.trim().toLowerCase(); + + if (XRY_NAMESPACES.contains(normalizedXryLine)) { + namespace = xryLine.trim(); + continue; + } + + //Find the XRY key on this line. + int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); + if (keyDelimiter == -1) { + logger.log(Level.SEVERE, String.format("XRY DSP: Expected a key value " + + "pair on this line (in brackets) [ %s ], but one was not detected." + + " Is this the continuation of a previous line?" + + " Here is the previous line (in brackets) [ %s ]. " + + "What does this key mean?", xryLine, xryLines[i - 1])); + continue; + } + + //Extract the key value pair + String key = xryLine.substring(0, keyDelimiter).trim(); + String value = xryLine.substring(keyDelimiter + 1).trim(); + + String normalizedKey = key.toLowerCase(); + + if (XRY_META_KEYS.contains(normalizedKey)) { + //Skip meta keys, they are being dealt with seperately. + continue; + } + + if (!XRY_KEYS.contains(normalizedKey)) { + logger.log(Level.SEVERE, String.format("XRY DSP: The following key, " + + "value pair (in brackets, respectively) [ %s ], [ %s ] " + + "was not recognized. Discarding... Here is the previous line " + + "[ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); + continue; + } + + if (value.isEmpty()) { + logger.log(Level.SEVERE, String.format("XRY DSP: The following key " + + "(in brackets) [ %s ] was recognized, but the value " + + "was empty. Discarding... Here is the previous line " + + "for context [ %s ]. Is this a continuation of this line? " + + "What does an empty key mean?", key, xryLines[i - 1])); + continue; + } + + //Assume text is the only field that can span multiple lines. + if (normalizedKey.equals(TEXT_KEY)) { + //Build up multiple lines. + for (; i + 1 < xryLines.length + && !hasKey(xryLines[i + 1]) + && !hasNamespace(xryLines[i + 1]); i++) { + String continuedValue = xryLines[i + 1].trim(); + //Assume multi lined values are split by word. + value = value + " " + continuedValue; + } + + int referenceNumber = getMetaInfo(xryLines, REFERENCE_NUMBER); + //Check if there is any segmented text. Min val is used to + //signify that no reference number was found. + if (referenceNumber != Integer.MIN_VALUE) { + logger.log(Level.INFO, String.format("XRY DSP: Message entity " + + "appears to be segmented with reference number [ %d ]", referenceNumber)); + + if (referenceNumbersSeen.contains(referenceNumber)) { + logger.log(Level.SEVERE, "XRY DSP: This reference has already " + + "been seen. This means that the segments are not " + + "contiguous. Any segments contiguous with this " + + "one will be aggregated and another " + + "(otherwise duplicate) artifact will be created."); + } + + referenceNumbersSeen.add(referenceNumber); + + int segmentNumber = getMetaInfo(xryLines, SEGMENT_NUMBER); + + //Unify segmented text, if there is any. + String segmentedText = getSegmentedText(referenceNumber, + segmentNumber, reader); + //Assume it was segmented by word. + value = value + " " + segmentedText; + } + } + + BlackboardAttribute attribute = makeAttribute(namespace, key, value); + if (attribute != null) { + attributes.add(attribute); + } + } + + //Only create artifacts with non-empty attributes. + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); + artifact.addAttributes(attributes); + } + } + } + + /** + * + * @param referenceNumber + * @param segmentNumber + * @param reader + * @return + * @throws IOException + */ + private String getSegmentedText(int referenceNumber, int segmentNumber, XRYFileReader reader) throws IOException { + StringBuilder segmentedText = new StringBuilder(); + + while (reader.hasNextEntity()) { + //Peek at the next to see if it has the same reference number. + String nextEntity = reader.peek(); + String[] nextEntityLines = nextEntity.split("\n"); + int nextReferenceNumber = getMetaInfo(nextEntityLines, REFERENCE_NUMBER); + + if (nextReferenceNumber != referenceNumber) { + //Don't consume the next entity. It is not related + //to the current message thread. + break; + } + + //Consume the entity. + reader.nextEntity(); + + int nextSegmentNumber = getMetaInfo(nextEntityLines, SEGMENT_NUMBER); + + //Extract the text key from the entity, which is potentially + //multi-lined. + if (nextEntityLines.length > 0) { + logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ] " + + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber)); + } + + if (nextSegmentNumber != segmentNumber + 1) { + logger.log(Level.SEVERE, String.format("XRY DSP: Contiguous " + + "segments are not ascending incrementally. Encountered " + + "segment [ %d ] after segment [ %d ]. This means the reconstructed " + + "text will be out of order.", nextSegmentNumber, segmentNumber)); + } + + for (int i = 1; i < nextEntityLines.length; i++) { + String xryLine = nextEntityLines[i]; + //Find the XRY key on this line. + int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); + if (keyDelimiter == -1) { + //Skip this line, we are searching only for a text key-value pair. + continue; + } + + String key = xryLine.substring(0, keyDelimiter); + String normalizedKey = key.trim().toLowerCase(); + if (normalizedKey.equals(TEXT_KEY)) { + String value = xryLine.substring(keyDelimiter + 1).trim(); + segmentedText.append(value).append(' '); + + //Build up multiple lines. + for (; (i + 1) < nextEntityLines.length + && !hasKey(nextEntityLines[i + 1]) + && !hasNamespace(nextEntityLines[i + 1]); i++) { + String continuedValue = nextEntityLines[i + 1].trim(); + segmentedText.append(continuedValue).append(' '); + } + } + } + + segmentNumber = nextSegmentNumber; + } + + //Remove the trailing space. + if (segmentedText.length() > 0) { + segmentedText.setLength(segmentedText.length() - 1); + } + return segmentedText.toString(); + } + + /** + * Determines if the line has recognized key value on it. + * + * @param xryLine + * @return + */ + private boolean hasKey(String xryLine) { + int delimiter = xryLine.indexOf(':'); + if (delimiter != -1) { + String key = xryLine.substring(0, delimiter); + String normalizedKey = key.trim().toLowerCase(); + return XRY_KEYS.contains(normalizedKey); + } else { + return false; + } + } + + /** + * Determines if the line is a recognized namespace. + * + * @param xryLine + * @return + */ + private boolean hasNamespace(String xryLine) { + String normalizedLine = xryLine.trim().toLowerCase(); + return XRY_NAMESPACES.contains(normalizedLine); + } + + /** + * Extracts meta keys from the XRY entity. All of the known meta + * keys are integers and describe the message segments. + * + * @param xryLines Current XRY entity + * @param expectedKey The meta key to search for + * @return The interpreted integer value or Integer.MIN_VALUE if + * no meta key was found. + */ + private int getMetaInfo(String[] xryLines, String metaKey) { + for (int i = 0; i < xryLines.length; i++) { + String xryLine = xryLines[i]; + + String normalizedXryLine = xryLine.trim().toLowerCase(); + int firstDelimiter = normalizedXryLine.indexOf(KEY_VALUE_DELIMITER); + if (firstDelimiter != -1) { + String key = normalizedXryLine.substring(0, firstDelimiter); + if (key.equals(metaKey)) { + String value = normalizedXryLine.substring(firstDelimiter + 1).trim(); + try { + return Integer.parseInt(value); + } catch (NumberFormatException ex) { + logger.log(Level.SEVERE, String.format("XRY DSP: Value [ %s ] for " + + "meta key [ %s ] was not an integer.", value, metaKey), ex); + } + } + } + } + + return Integer.MIN_VALUE; + } + + /** + * Creates an attribute from the extracted key value pair. + * + * @param nameSpace The namespace of this key value pair. + * It will have been verified beforehand, otherwise it will be empty. + * @param key The key that was verified beforehand + * @param value The value associated with that key. + * @return + */ + private BlackboardAttribute makeAttribute(String namespace, String key, String value) { + String normalizedKey = key.toLowerCase(); + String normalizedNamespace = namespace.toLowerCase(); + + switch (normalizedKey) { + case "time": + //Tranform value to epoch ms + try { + String dateTime = removeDateTimeLocale(value); + String normalizedDateTime = dateTime.trim(); + long dateTimeInEpoch = calculateMsSinceEpoch(normalizedDateTime); + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, PARSER_NAME, dateTimeInEpoch); + } catch (DateTimeParseException ex) { + logger.log(Level.SEVERE, String.format("XRY DSP: Assumption " + + "about the date time formatting of messages is not " + + "right. Here is the value [ %s ].", value), ex); + return null; + } + case "direction": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, PARSER_NAME, value); + case "text": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT, PARSER_NAME, value); + case "status": + String normalizedValue = value.toLowerCase(); + switch (normalizedValue) { + case "read": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, READ); + case "unread": + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, UNREAD); + case "sending failed": + //Ignore for now. + return null; + case "deleted": + //Ignore for now. + return null; + case "unsent": + //Ignore for now. + return null; + default: + logger.log(Level.SEVERE, String.format("XRY DSP: Unrecognized " + + "status value [ %s ].", value)); + return null; + } + case "type": + //Ignore for now. + return null; + case "storage": + //Ignore for now. + return null; + case "index": + //Ignore for now. + return null; + case "folder": + //Ignore for now. + return null; + case "service center": + //Ignore for now. + return null; + case "tel": + //Apply the namespace + if (normalizedNamespace.equals("from")) { + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, PARSER_NAME, value); + } else { + //Assume to and participant are both equivalent to TSK_PHONE_NUMBER_TO + return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, PARSER_NAME, value); + } + default: + throw new IllegalArgumentException(String.format("key [ %s ] was not recognized.", key)); + } + } + + /** + * Removes the locale from the date time value. + * + * Locale in this case being (Device) or (Network). + * + * @param dateTime XRY datetime value to be sanitized. + * @return A purer date time value. + */ + private String removeDateTimeLocale(String dateTime) { + int index = dateTime.indexOf('('); + if (index == -1) { + return dateTime; + } + + return dateTime.substring(0, index); + } + + /** + * Parses the date time value and calculates ms since epoch. The time zone is + * assumed to be UTC. + * + * @param dateTime + * @return + */ + private long calculateMsSinceEpoch(String dateTime) { + LocalDateTime localDateTime = LocalDateTime.parse(dateTime, DATE_TIME_PARSER); + //Assume dates have no offset. + return localDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index 3c6e881ada..a7443e7f48 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -31,9 +31,6 @@ import org.sleuthkit.datamodel.TskCoreException; */ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { - //Human readable name of this parser. - private static final String PARSER_NAME = "XRY Web Bookmarks"; - //All known XRY keys for web bookmarks. private static final Map KEY_TO_TYPE = new HashMap() { From 875b789abb8439a1b95a12fb5c56cfaa56bd5c7a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 13 Nov 2019 16:26:02 -0500 Subject: [PATCH 11/12] Update comments, add some additional logging, and address some codacy comments --- .../xry/XRYCallsFileParser.java | 2 + .../xry/XRYMessagesFileParser.java | 42 ++++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index af59c6891d..260fccd40f 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -119,8 +119,10 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { case "received": return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, PARSER_NAME, INCOMING); case "dialed": + //Ignore for now. return null; case "last dialed": + //Ignore for now. return null; default: logger.log(Level.SEVERE, String.format("Call type (in brackets) [ %s ] not recognized.", value)); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java index c1a80f59ae..253676f0e9 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -74,6 +74,7 @@ final class XRYMessagesFileParser implements XRYFileParser { add("folder"); add("service center"); add("type"); + add("name"); } }; @@ -100,12 +101,12 @@ final class XRYMessagesFileParser implements XRYFileParser { * attributes can span multiple lines. The "Text" key is the only known key * value pair that can span multiple lines. Messages can be segmented, * meaning that their "Text" content can appear in multiple XRY entities. - * Our goal for a segmented message is aggregate all of the text pieces and + * Our goal for a segmented message is to aggregate all of the text pieces and * create 1 artifact. * * This parse implementation assumes that segments are contiguous and that * they ascend incrementally. There are checks in place to verify this - * assumption are correct, otherwise an error will appear in the logs. + * assumption is correct, otherwise an error will appear in the logs. * * @param reader The XRYFileReader that reads XRY entities from the * Message-SMS report. @@ -235,9 +236,11 @@ final class XRYMessagesFileParser implements XRYFileParser { } /** - * - * @param referenceNumber - * @param segmentNumber + * Builds up segmented message entities so that the text is unified in the + * artifact. + * + * @param referenceNumber Reference number that messages are group by + * @param segmentNumber Segment number of the starting segment. * @param reader * @return * @throws IOException @@ -245,6 +248,7 @@ final class XRYMessagesFileParser implements XRYFileParser { private String getSegmentedText(int referenceNumber, int segmentNumber, XRYFileReader reader) throws IOException { StringBuilder segmentedText = new StringBuilder(); + int currentSegmentNumber = segmentNumber; while (reader.hasNextEntity()) { //Peek at the next to see if it has the same reference number. String nextEntity = reader.peek(); @@ -269,11 +273,11 @@ final class XRYMessagesFileParser implements XRYFileParser { + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber)); } - if (nextSegmentNumber != segmentNumber + 1) { + if (nextSegmentNumber != currentSegmentNumber + 1) { logger.log(Level.SEVERE, String.format("XRY DSP: Contiguous " + "segments are not ascending incrementally. Encountered " + "segment [ %d ] after segment [ %d ]. This means the reconstructed " - + "text will be out of order.", nextSegmentNumber, segmentNumber)); + + "text will be out of order.", nextSegmentNumber, currentSegmentNumber)); } for (int i = 1; i < nextEntityLines.length; i++) { @@ -301,7 +305,7 @@ final class XRYMessagesFileParser implements XRYFileParser { } } - segmentNumber = nextSegmentNumber; + currentSegmentNumber = nextSegmentNumber; } //Remove the trailing space. @@ -383,6 +387,7 @@ final class XRYMessagesFileParser implements XRYFileParser { private BlackboardAttribute makeAttribute(String namespace, String key, String value) { String normalizedKey = key.toLowerCase(); String normalizedNamespace = namespace.toLowerCase(); + String normalizedValue = value.toLowerCase(); switch (normalizedKey) { case "time": @@ -403,7 +408,6 @@ final class XRYMessagesFileParser implements XRYFileParser { case "text": return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT, PARSER_NAME, value); case "status": - String normalizedValue = value.toLowerCase(); switch (normalizedValue) { case "read": return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, READ); @@ -424,8 +428,21 @@ final class XRYMessagesFileParser implements XRYFileParser { return null; } case "type": - //Ignore for now. - return null; + switch (normalizedValue) { + case "deliver": + //Ignore for now. + return null; + case "submit": + //Ignore for now. + return null; + case "status report": + //Ignore for now. + return null; + default: + logger.log(Level.SEVERE, String.format("XRY DSP: Unrecognized " + + "type value [ %s ]", value)); + return null; + } case "storage": //Ignore for now. return null; @@ -435,6 +452,9 @@ final class XRYMessagesFileParser implements XRYFileParser { case "folder": //Ignore for now. return null; + case "name": + //Ignore for now. + return null; case "service center": //Ignore for now. return null; From 53b134daedb6e90d660ec409fb9a4943061bfeda Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 14 Nov 2019 14:23:59 -0500 Subject: [PATCH 12/12] Changed the log messages to be easier to read and fixed the date time bug --- .../xry/AbstractSingleKeyValueParser.java | 10 +++--- .../xry/XRYCallsFileParser.java | 8 ++--- .../xry/XRYDeviceGenInfoFileParser.java | 12 +++---- .../xry/XRYFileParserFactory.java | 2 +- .../xry/XRYMessagesFileParser.java | 36 +++++++++---------- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java index 20b6b7c1bf..edd218adbb 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -45,7 +45,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { @Override public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { Path reportPath = reader.getReportPath(); - logger.log(Level.INFO, String.format("XRY DSP: Processing report at [ %s ]", reportPath.toString())); + logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString())); while (reader.hasNextEntity()) { String xryEntity = reader.nextEntity(); @@ -55,7 +55,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { //First line of the entity is the title. if (xryLines.length > 0) { - logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ]", xryLines[0])); + logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); } String namespace = ""; @@ -75,7 +75,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { //the start of the line and the first delimiter. int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); if (keyDelimiter == -1) { - logger.log(Level.SEVERE, String.format("XRY DSP: Expected a key value " + logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value " + "pair on this line (in brackets) [ %s ], but one was not detected." + " Here is the previous line [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); continue; @@ -84,14 +84,14 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { String value = xryLine.substring(keyDelimiter + 1).trim(); if (!isKey(key)) { - logger.log(Level.SEVERE, String.format("XRY DSP: The following key, " + logger.log(Level.SEVERE, String.format("[XRY DSP] The following key, " + "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..." + " Here is the previous line [ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); continue; } if (value.isEmpty()) { - logger.log(Level.SEVERE, String.format("XRY DSP: The following key " + logger.log(Level.SEVERE, String.format("[XRY DSP] The following key " + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding..." + " Here is the previous line for context [ %s ]. What does this mean?", key, xryLines[i - 1])); continue; diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 260fccd40f..596f25e7b1 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -89,10 +89,10 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { try { String dateTime = removeDateTimeLocale(value); String normalizedDateTime = dateTime.trim(); - long dateTimeInEpoch = calculateMsSinceEpoch(normalizedDateTime); + long dateTimeInEpoch = calculateSecondsSinceEpoch(normalizedDateTime); return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeInEpoch); } catch (DateTimeParseException ex) { - logger.log(Level.SEVERE, String.format("XRY DSP: Assumption about the date time " + logger.log(Level.SEVERE, String.format("[XRY DSP] Assumption about the date time " + "formatting of call logs is not right. Here is the value [ %s ]", value), ex); return null; } @@ -167,9 +167,9 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @param dateTime * @return */ - private long calculateMsSinceEpoch(String dateTime) { + private long calculateSecondsSinceEpoch(String dateTime) { LocalDateTime localDateTime = LocalDateTime.parse(dateTime, DATE_TIME_PARSER); //Assume dates have no offset. - return localDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + return localDateTime.toInstant(ZoneOffset.UTC).getEpochSecond(); } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index d610ffb89d..d3bba45bfc 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -83,7 +83,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { @Override public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { Path reportPath = reader.getReportPath(); - logger.log(Level.INFO, String.format("XRY DSP: Processing report at [ %s ]", reportPath.toString())); + logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString())); while (reader.hasNextEntity()) { String xryEntity = reader.nextEntity(); @@ -93,7 +93,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //First line of the entity is the title. if (xryLines.length > 0) { - logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ]", xryLines[0])); + logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); } for (int i = 1; i < xryLines.length; i++) { @@ -101,7 +101,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //Expecting to see a "Data" key. if (!hasDataKey(xryLine)) { - logger.log(Level.SEVERE, String.format("XRY DSP: Expected a 'Data' key " + logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a 'Data' key " + "on this line (in brackets) [ %s ], but none was found. " + "Discarding... Here is the previous line for context [ %s ]. " + "What does this mean?", xryLine, xryLines[i - 1])); @@ -109,7 +109,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { } if (i + 1 == xryLines.length) { - logger.log(Level.SEVERE, String.format("XRY DSP: Found a 'Data' key " + logger.log(Level.SEVERE, String.format("[XRY DSP] Found a 'Data' key " + "but no corresponding 'Attribute' key. Discarding... Here " + "is the 'Data' line (in brackets) [ %s ]. Here is the previous " + "line for context [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); @@ -123,7 +123,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //Expecting to see an "Attribute" key if (!hasAttributeKey(nextXryLine)) { - logger.log(Level.SEVERE, String.format("XRY DSP: Expected an 'Attribute' " + logger.log(Level.SEVERE, String.format("[XRY DSP] Expected an 'Attribute' " + "key on this line (in brackets) [ %s ], but none was found. " + "Discarding... Here is the previous line for context [ %s ]. " + "What does this mean?", nextXryLine, xryLine)); @@ -139,7 +139,7 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //All of the attribute types in the map expect a string. attributes.add(new BlackboardAttribute(KEY_TO_TYPE.get(normalizedAttributeValue), PARSER_NAME, dataValue)); } else { - logger.log(Level.SEVERE, String.format("XRY DSP: Attribute type (in brackets) " + logger.log(Level.SEVERE, String.format("[XRY DSP] Attribute type (in brackets) " + "[ %s ] was not recognized. Discarding... Here is the " + "previous line for context [ %s ]. What does this mean?", nextXryLine, xryLine)); } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java index d650510827..06492de07b 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java @@ -42,7 +42,7 @@ final class XRYFileParserFactory { throw new IllegalArgumentException("Report type cannot be null"); } - switch (reportType.toLowerCase()) { + switch (reportType.trim().toLowerCase()) { case "calls": return new XRYCallsFileParser(); case "contacts/contacts": diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java index 253676f0e9..ac78e62509 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -118,7 +118,7 @@ final class XRYMessagesFileParser implements XRYFileParser { @Override public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { Path reportPath = reader.getReportPath(); - logger.log(Level.INFO, String.format("XRY DSP: Processing report at [ %s ]", reportPath.toString())); + logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString())); //Keep track of the reference numbers that have been parsed. Set referenceNumbersSeen = new HashSet<>(); @@ -129,7 +129,7 @@ final class XRYMessagesFileParser implements XRYFileParser { //First line of the entity is the title. if (xryLines.length > 0) { - logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ]", xryLines[0])); + logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); } List attributes = new ArrayList<>(); @@ -147,7 +147,7 @@ final class XRYMessagesFileParser implements XRYFileParser { //Find the XRY key on this line. int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); if (keyDelimiter == -1) { - logger.log(Level.SEVERE, String.format("XRY DSP: Expected a key value " + logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value " + "pair on this line (in brackets) [ %s ], but one was not detected." + " Is this the continuation of a previous line?" + " Here is the previous line (in brackets) [ %s ]. " @@ -167,7 +167,7 @@ final class XRYMessagesFileParser implements XRYFileParser { } if (!XRY_KEYS.contains(normalizedKey)) { - logger.log(Level.SEVERE, String.format("XRY DSP: The following key, " + logger.log(Level.SEVERE, String.format("[XRY DSP] The following key, " + "value pair (in brackets, respectively) [ %s ], [ %s ] " + "was not recognized. Discarding... Here is the previous line " + "[ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); @@ -175,7 +175,7 @@ final class XRYMessagesFileParser implements XRYFileParser { } if (value.isEmpty()) { - logger.log(Level.SEVERE, String.format("XRY DSP: The following key " + logger.log(Level.SEVERE, String.format("[XRY DSP] The following key " + "(in brackets) [ %s ] was recognized, but the value " + "was empty. Discarding... Here is the previous line " + "for context [ %s ]. Is this a continuation of this line? " @@ -186,7 +186,7 @@ final class XRYMessagesFileParser implements XRYFileParser { //Assume text is the only field that can span multiple lines. if (normalizedKey.equals(TEXT_KEY)) { //Build up multiple lines. - for (; i + 1 < xryLines.length + for (; (i + 1) < xryLines.length && !hasKey(xryLines[i + 1]) && !hasNamespace(xryLines[i + 1]); i++) { String continuedValue = xryLines[i + 1].trim(); @@ -198,15 +198,15 @@ final class XRYMessagesFileParser implements XRYFileParser { //Check if there is any segmented text. Min val is used to //signify that no reference number was found. if (referenceNumber != Integer.MIN_VALUE) { - logger.log(Level.INFO, String.format("XRY DSP: Message entity " + logger.log(Level.INFO, String.format("[XRY DSP] Message entity " + "appears to be segmented with reference number [ %d ]", referenceNumber)); if (referenceNumbersSeen.contains(referenceNumber)) { - logger.log(Level.SEVERE, "XRY DSP: This reference has already " + logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already " + "been seen. This means that the segments are not " + "contiguous. Any segments contiguous with this " + "one will be aggregated and another " - + "(otherwise duplicate) artifact will be created."); + + "(otherwise duplicate) artifact will be created.", referenceNumber)); } referenceNumbersSeen.add(referenceNumber); @@ -269,12 +269,12 @@ final class XRYMessagesFileParser implements XRYFileParser { //Extract the text key from the entity, which is potentially //multi-lined. if (nextEntityLines.length > 0) { - logger.log(Level.INFO, String.format("XRY DSP: Processing [ %s ] " + logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] " + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber)); } if (nextSegmentNumber != currentSegmentNumber + 1) { - logger.log(Level.SEVERE, String.format("XRY DSP: Contiguous " + logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous " + "segments are not ascending incrementally. Encountered " + "segment [ %d ] after segment [ %d ]. This means the reconstructed " + "text will be out of order.", nextSegmentNumber, currentSegmentNumber)); @@ -365,7 +365,7 @@ final class XRYMessagesFileParser implements XRYFileParser { try { return Integer.parseInt(value); } catch (NumberFormatException ex) { - logger.log(Level.SEVERE, String.format("XRY DSP: Value [ %s ] for " + logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for " + "meta key [ %s ] was not an integer.", value, metaKey), ex); } } @@ -395,10 +395,10 @@ final class XRYMessagesFileParser implements XRYFileParser { try { String dateTime = removeDateTimeLocale(value); String normalizedDateTime = dateTime.trim(); - long dateTimeInEpoch = calculateMsSinceEpoch(normalizedDateTime); + long dateTimeInEpoch = calculateSecondsSinceEpoch(normalizedDateTime); return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, PARSER_NAME, dateTimeInEpoch); } catch (DateTimeParseException ex) { - logger.log(Level.SEVERE, String.format("XRY DSP: Assumption " + logger.log(Level.SEVERE, String.format("[XRY DSP] Assumption " + "about the date time formatting of messages is not " + "right. Here is the value [ %s ].", value), ex); return null; @@ -423,7 +423,7 @@ final class XRYMessagesFileParser implements XRYFileParser { //Ignore for now. return null; default: - logger.log(Level.SEVERE, String.format("XRY DSP: Unrecognized " + logger.log(Level.SEVERE, String.format("[XRY DSP] Unrecognized " + "status value [ %s ].", value)); return null; } @@ -439,7 +439,7 @@ final class XRYMessagesFileParser implements XRYFileParser { //Ignore for now. return null; default: - logger.log(Level.SEVERE, String.format("XRY DSP: Unrecognized " + logger.log(Level.SEVERE, String.format("[XRY DSP] Unrecognized " + "type value [ %s ]", value)); return null; } @@ -495,9 +495,9 @@ final class XRYMessagesFileParser implements XRYFileParser { * @param dateTime * @return */ - private long calculateMsSinceEpoch(String dateTime) { + private long calculateSecondsSinceEpoch(String dateTime) { LocalDateTime localDateTime = LocalDateTime.parse(dateTime, DATE_TIME_PARSER); //Assume dates have no offset. - return localDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + return localDateTime.toInstant(ZoneOffset.UTC).getEpochSecond(); } }