1
0
mirror of https://github.com/elisspace/autopsy.git synced 2026-09-02 23:25:51 +00:00

Merge pull request #5412 from dannysmyda/5733-implement-XRY-report-parsing

5733 implement xry report parsing
This commit is contained in:
Richard Cordovano
2019-11-14 15:00:29 -05:00
committed by GitHub
10 changed files with 1366 additions and 11 deletions

View File

@@ -0,0 +1,165 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Template parse method for reports that make blackboard attributes from a
* single key value pair.
*
* This parse implementation will create 1 artifact per XRY entity.
*/
abstract class AbstractSingleKeyValueParser implements XRYFileParser {
private static final Logger logger = Logger.getLogger(AbstractSingleKeyValueParser.class.getName());
private static final char KEY_VALUE_DELIMITER = ':';
protected static final String PARSER_NAME = "XRY DSP";
@Override
public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException {
Path reportPath = reader.getReportPath();
logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString()));
while (reader.hasNextEntity()) {
String xryEntity = reader.nextEntity();
String[] xryLines = xryEntity.split("\n");
List<BlackboardAttribute> attributes = new ArrayList<>();
//First line of the entity is the title.
if (xryLines.length > 0) {
logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
}
String namespace = "";
//Process each line, searching for a key value pair or a namespace.
//If neither are found, an error message is logged.
for (int i = 1; i < xryLines.length; i++) {
String xryLine = xryLines[i];
//Check if the line is a namespace, which gives context to the keys
//that follow.
if (isNamespace(xryLine)) {
namespace = xryLine.trim();
continue;
}
//Find the XRY key on this line. Assume key is the value between
//the start of the line and the first delimiter.
int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER);
if (keyDelimiter == -1) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value "
+ "pair on this line (in brackets) [ %s ], but one was not detected."
+ " Here is the previous line [ %s ]. What does this mean?", xryLine, xryLines[i - 1]));
continue;
}
String key = xryLine.substring(0, keyDelimiter).trim();
String value = xryLine.substring(keyDelimiter + 1).trim();
if (!isKey(key)) {
logger.log(Level.SEVERE, String.format("[XRY DSP] The following key, "
+ "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..."
+ " Here is the previous line [ %s ] for context. What does this key mean?", key, value, xryLines[i - 1]));
continue;
}
if (value.isEmpty()) {
logger.log(Level.SEVERE, String.format("[XRY DSP] The following key "
+ "(in brackets) [ %s ] was recognized, but the value was empty. Discarding..."
+ " Here is the previous line for context [ %s ]. What does this mean?", key, xryLines[i - 1]));
continue;
}
BlackboardAttribute attribute = makeAttribute(namespace, key, value);
//Temporarily allowing null to be valid return type until a decision
//is made about how to handle keys we are choosing to ignore.
if (attribute != null) {
attributes.add(makeAttribute(namespace, key, value));
}
}
//Only create artifacts with non-empty attributes.
if (!attributes.isEmpty()) {
makeArtifact(attributes, parent);
}
}
}
/**
* Determines if the key candidate is a known key. A key candidate is a
* string literal that begins a line and is terminated by a semi-colon.
*
* Ex:
*
* Call Type : Missed
*
* "Call Type" would be the key candidate that was extracted.
*
* @param key Key to test. These keys are trimmed of whitespace only.
* @return Indication if this key can be processed.
*/
abstract boolean isKey(String key);
/**
* Determines if the namespace candidate is a known namespace. A namespace
* candidate is a string literal that makes up an entire line.
*
* Ex:
*
* To
* Tel : +1245325
*
* "To" would be the candidate namespace that was extracted.
*
* @param nameSpace Namespace to test. Namespaces are trimmed of whitespace
* only.
* @return Indication if this namespace can be processed.
*/
abstract boolean isNamespace(String nameSpace);
/**
* Creates an attribute from the extracted key value pair.
*
* @param nameSpace The namespace of this key value pair.
* It will have been verified with isNamespace, otherwise it will be empty.
* @param key The key that was verified with isKey.
* @param value The value associated with that key.
* @return
*/
abstract BlackboardAttribute makeAttribute(String nameSpace, String key, String value);
/**
* Makes an artifact from the parsed attributes.
*
* @return
*/
abstract void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException;
}

View File

@@ -0,0 +1,175 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Parses XRY Calls files and creates artifacts.
*/
final class XRYCallsFileParser extends AbstractSingleKeyValueParser {
private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName());
private static final DateTimeFormatter DATE_TIME_PARSER
= DateTimeFormatter.ofPattern("M/d/y h:m:s [a][ z]");
private static final String INCOMING = "Incoming";
//All known XRY keys for call reports.
private static final Set<String> XRY_KEYS = new HashSet<String>() {
{
add("tel");
add("number");
add("call type");
add("name (matched)");
add("time");
add("duration");
add("storage");
add("index");
}
};
//All known XRY namespaces for call reports.
private static final Set<String> XRY_NAMESPACES = new HashSet<String>() {
{
add("to");
add("from");
}
};
@Override
boolean isKey(String key) {
String normalizedKey = key.toLowerCase();
return XRY_KEYS.contains(normalizedKey);
}
@Override
boolean isNamespace(String nameSpace) {
String normalizedNamespace = nameSpace.toLowerCase();
return XRY_NAMESPACES.contains(normalizedNamespace);
}
@Override
BlackboardAttribute makeAttribute(String nameSpace, String key, String value) {
String normalizedKey = key.toLowerCase();
String normalizedNamespace = nameSpace.toLowerCase();
switch (normalizedKey) {
case "time":
//Tranform value to epoch ms
try {
String dateTime = removeDateTimeLocale(value);
String normalizedDateTime = dateTime.trim();
long dateTimeInEpoch = calculateSecondsSinceEpoch(normalizedDateTime);
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeInEpoch);
} catch (DateTimeParseException ex) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Assumption about the date time "
+ "formatting of call logs is not right. Here is the value [ %s ]", value), ex);
return null;
}
case "duration":
//Ignore for now.
return null;
case "storage":
//Ignore for now.
return null;
case "index":
//Ignore for now.
return null;
case "tel":
//Apply the namespace
if(normalizedNamespace.equals("from")) {
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, PARSER_NAME, value);
} else {
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, PARSER_NAME, value);
}
case "call type":
String normalizedValue = value.toLowerCase();
switch (normalizedValue) {
case "missed":
case "received":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, PARSER_NAME, INCOMING);
case "dialed":
//Ignore for now.
return null;
case "last dialed":
//Ignore for now.
return null;
default:
logger.log(Level.SEVERE, String.format("Call type (in brackets) [ %s ] not recognized.", value));
return null;
}
case "number":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, value);
case "name (matched)":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, PARSER_NAME, value);
default:
throw new IllegalArgumentException(String.format("key [ %s ] was not recognized.", key));
}
}
@Override
void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException {
BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG);
artifact.addAttributes(attributes);
}
/**
* Removes the locale from the date time value.
*
* Locale in this case being (Device) or (Network).
*
* @param dateTime XRY datetime value to be sanitized.
* @return A purer date time value.
*/
private String removeDateTimeLocale(String dateTime) {
int index = dateTime.indexOf('(');
if (index == -1) {
return dateTime;
}
return dateTime.substring(0, index);
}
/**
* Parses the date time value and calculates ms since epoch. The time zone is
* assumed to be UTC.
*
* @param dateTime
* @return
*/
private long calculateSecondsSinceEpoch(String dateTime) {
LocalDateTime localDateTime = LocalDateTime.parse(dateTime, DATE_TIME_PARSER);
//Assume dates have no offset.
return localDateTime.toInstant(ZoneOffset.UTC).getEpochSecond();
}
}

View File

@@ -0,0 +1,74 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Parses XRY Contacts-Contacts files and creates artifacts.
*/
final class XRYContactsFileParser extends AbstractSingleKeyValueParser {
//All of the known XRY keys for contacts.
private static final Set<String> XRY_KEYS = new HashSet<String>() {{
add("name");
add("tel");
add("storage");
}};
@Override
boolean isKey(String key) {
String normalizedKey = key.toLowerCase();
return XRY_KEYS.contains(normalizedKey);
}
@Override
boolean isNamespace(String nameSpace) {
//No namespaces are currently known for this report type.
return false;
}
@Override
BlackboardAttribute makeAttribute(String nameSpace, String key, String value) {
String normalizedKey = key.toLowerCase();
switch(normalizedKey) {
case "name":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, PARSER_NAME, value);
case "tel":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, value);
case "storage":
//Ignore for now.
return null;
default:
throw new IllegalArgumentException(String.format("Key [ %s ] was not recognized", key));
}
}
@Override
void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException {
BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT);
artifact.addAttributes(attributes);
}
}

View File

@@ -0,0 +1,191 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Parses XRY Device-General Information files and creates artifacts.
*/
final class XRYDeviceGenInfoFileParser implements XRYFileParser {
private static final Logger logger = Logger.getLogger(XRYDeviceGenInfoFileParser.class.getName());
//Human readable name of this parser.
private static final String PARSER_NAME = "XRY DSP";
private static final char KEY_VALUE_DELIMITER = ':';
//All known XRY keys for Device Gen Info reports.
private static final String ATTRIBUTE_KEY = "attribute";
private static final String DATA_KEY = "data";
//All of the known XRY Attribute values for device gen info. The value of the
//attribute keys are actionable for this parser. See parse header for more
//details.
private static final Map<String, BlackboardAttribute.ATTRIBUTE_TYPE> KEY_TO_TYPE
= new HashMap<String, BlackboardAttribute.ATTRIBUTE_TYPE>() {
{
put("device name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_NAME);
put("device family", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MODEL);
put("device type", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MAKE);
put("mobile id (imei)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI);
put("security code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD);
}
};
/**
* Device-General Information reports have 2 key value pairs for every
* attribute. The two only known keys are "Data" and "Attribute", where data
* is some generic information that the Attribute key describes.
*
* Example:
*
* Data: Nokia XYZ
* Attribute: Device Name
*
* This parse implementation assumes that the data field does not span
* multiple lines. If the data does span multiple lines, it will log an
* error describing an expectation for an "Attribute" key that is not found.
*
* @param reader The XRYFileReader that reads XRY entities from the
* Device-General Information report.
* @param parent The parent Content to create artifacts from.
* @throws IOException If an I/O error is encountered during report reading
* @throws TskCoreException If an error during artifact creation is encountered.
*/
@Override
public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException {
Path reportPath = reader.getReportPath();
logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString()));
while (reader.hasNextEntity()) {
String xryEntity = reader.nextEntity();
String[] xryLines = xryEntity.split("\n");
List<BlackboardAttribute> attributes = new ArrayList<>();
//First line of the entity is the title.
if (xryLines.length > 0) {
logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
}
for (int i = 1; i < xryLines.length; i++) {
String xryLine = xryLines[i];
//Expecting to see a "Data" key.
if (!hasDataKey(xryLine)) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a 'Data' key "
+ "on this line (in brackets) [ %s ], but none was found. "
+ "Discarding... Here is the previous line for context [ %s ]. "
+ "What does this mean?", xryLine, xryLines[i - 1]));
continue;
}
if (i + 1 == xryLines.length) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Found a 'Data' key "
+ "but no corresponding 'Attribute' key. Discarding... Here "
+ "is the 'Data' line (in brackets) [ %s ]. Here is the previous "
+ "line for context [ %s ]. What does this mean?", xryLine, xryLines[i - 1]));
continue;
}
int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER);
String dataValue = xryLine.substring(dataKeyIndex + 1).trim();
String nextXryLine = xryLines[++i];
//Expecting to see an "Attribute" key
if (!hasAttributeKey(nextXryLine)) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Expected an 'Attribute' "
+ "key on this line (in brackets) [ %s ], but none was found. "
+ "Discarding... Here is the previous line for context [ %s ]. "
+ "What does this mean?", nextXryLine, xryLine));
continue;
}
int attributeKeyIndex = nextXryLine.indexOf(KEY_VALUE_DELIMITER);
String attributeValue = nextXryLine.substring(attributeKeyIndex + 1).trim();
String normalizedAttributeValue = attributeValue.toLowerCase();
//Check if the attribute value is recognized.
if (KEY_TO_TYPE.containsKey(normalizedAttributeValue)) {
//All of the attribute types in the map expect a string.
attributes.add(new BlackboardAttribute(KEY_TO_TYPE.get(normalizedAttributeValue), PARSER_NAME, dataValue));
} else {
logger.log(Level.SEVERE, String.format("[XRY DSP] Attribute type (in brackets) "
+ "[ %s ] was not recognized. Discarding... Here is the "
+ "previous line for context [ %s ]. What does this mean?", nextXryLine, xryLine));
}
}
if(!attributes.isEmpty()) {
//Build the artifact.
BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO);
artifact.addAttributes(attributes);
}
}
}
/**
* Determines if the XRY line has a data key on it.
*
* @param xryLine
* @return
*/
private boolean hasDataKey(String xryLine) {
int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER);
//No key structure found.
if (dataKeyIndex == -1) {
return false;
}
String normalizedDataKey = xryLine.substring(0,
dataKeyIndex).trim().toLowerCase();
return normalizedDataKey.equals(DATA_KEY);
}
/**
* Determines if the XRY line has an attribute key on it.
*
* @param xryLine
* @return
*/
private boolean hasAttributeKey(String xryLine) {
int attributeKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER);
//No key structure found.
if (attributeKeyIndex == -1) {
return false;
}
String normalizedDataKey = xryLine.substring(0,
attributeKeyIndex).trim().toLowerCase();
return normalizedDataKey.equals(ATTRIBUTE_KEY);
}
}

View File

@@ -0,0 +1,46 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.io.IOException;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Interface for XRY file parsing.
*/
interface XRYFileParser {
/**
* Parses XRY entities and creates artifacts from the interpreted content.
*
* See XRYFileReader for more information on XRY entities. It is expected
* that implementations will create artifacts on the supplied Content
* object.
*
* @param reader Produces XRY entities from a given XRY file.
* @param parent Content object that will act as the source of the
* artifacts.
* @throws IOException If an I/O error occurs during reading.
* @throws TskCoreException If an error occurs during artifact creation.
*/
void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException;
}

View File

@@ -0,0 +1,80 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
/**
* Instantiates XRYFileParsers by report type.
*/
final class XRYFileParserFactory {
/**
* Creates the correct implementation of a XRYFileParser for the specified
* report type.
*
* It is assumed that the report type is supported, which means the client
* needs to have tested with supports beforehand. Otherwise, an
* IllegalArgumentException is thrown.
*
* @param reportType A supported XRY report type.
* @return A XRYFileParser with defined behavior for the report type.
* @throws IllegalArgumentException if the report type is not supported or
* is null. This is a misuse of the API. It is assumed that the report type
* has been tested with the supports method.
*/
static XRYFileParser get(String reportType) {
if (reportType == null) {
throw new IllegalArgumentException("Report type cannot be null");
}
switch (reportType.trim().toLowerCase()) {
case "calls":
return new XRYCallsFileParser();
case "contacts/contacts":
return new XRYContactsFileParser();
case "device/general information":
return new XRYDeviceGenInfoFileParser();
case "messages/sms":
return new XRYMessagesFileParser();
case "web/bookmarks":
return new XRYWebBookmarksFileParser();
default:
throw new IllegalArgumentException(reportType + " not recognized.");
}
}
/**
* Tests if a XRYFileParser implementation exists for the report type.
*
* @param reportType Report type to test.
* @return Indication if the report type can be parsed.
*/
static boolean supports(String reportType) {
try {
//Attempt a get.
get(reportType);
return true;
} catch (IllegalArgumentException ex) {
return false;
}
}
//Prevent direct instantiation
private XRYFileParserFactory() {
}
}

View File

@@ -45,36 +45,43 @@ import org.apache.commons.io.FilenameUtils;
* From
* Tel: 12345678
*/
public final class XRYFileReader implements AutoCloseable {
final class XRYFileReader implements AutoCloseable {
private static final Logger logger = Logger.getLogger(XRYFileReader.class.getName());
//Assume UTF_16LE
private static final Charset CHARSET = StandardCharsets.UTF_16LE;
//Assume TXT extension
private static final String EXTENSION = "txt";
//Assume 0xFFFE is the BOM
private static final int[] BOM = {0xFF, 0xFE};
//Assume all XRY reports have the type on the 3rd line.
private static final int LINE_WITH_REPORT_TYPE = 3;
//Assume all headers are 5 lines in length.
private static final int HEADER_LENGTH_IN_LINES = 5;
//Assume TXT extension
private static final String EXTENSION = "txt";
//Assume 0xFFFE is the BOM
private static final int[] BOM = {0xFF, 0xFE};
//Entity to be consumed during file iteration.
private final StringBuilder xryEntity;
//Underlying reader for the xry file.
private final BufferedReader reader;
private final StringBuilder xryEntity;
//Reference to the original xry file.
private final Path xryFilePath;
/**
* Creates an XRYFileReader. As part of construction, the XRY file is opened
* and the reader is advanced past the header. This leaves the reader
* positioned at the start of the first XRY entity.
*
* The file is assumed to be encoded in UTF-16LE.
* The file is assumed to be encoded in UTF-16LE and is NOT verified to be
* an XRY file before reading. It is expected that the isXRYFile function
* has been called on the path beforehand. Otherwise, the behavior is
* undefined.
*
* @param xryFile XRY file to read. It is assumed that the caller has read
* access to the path.
@@ -82,6 +89,7 @@ public final class XRYFileReader implements AutoCloseable {
*/
public XRYFileReader(Path xryFile) throws IOException {
reader = Files.newBufferedReader(xryFile, CHARSET);
xryFilePath = xryFile;
//Advance the reader to the start of the first XRY entity.
for (int i = 0; i < HEADER_LENGTH_IN_LINES; i++) {
@@ -91,6 +99,35 @@ public final class XRYFileReader implements AutoCloseable {
xryEntity = new StringBuilder();
}
/**
* Extracts the report type from the XRY file.
*
* @return The XRY report type
* @throws IOException if an I/O error occurs.
* @throws IllegalArgumentExcepton If the XRY file does not have a report
* type. This is a misuse of the API. The validity of the Path should have
* been checked with isXRYFile before creating an XRYFileReader.
*/
public String getReportType() throws IOException {
Optional<String> reportType = getType(xryFilePath);
if (reportType.isPresent()) {
return reportType.get();
}
throw new IllegalArgumentException(xryFilePath.toString() + " does not "
+ "have a report type.");
}
/**
* Returns the raw path of the XRY report file.
*
* @return
* @throws IOException
*/
public Path getReportPath() throws IOException {
return xryFilePath;
}
/**
* Advances the reader until a valid XRY entity is detected or EOF is
* reached.
@@ -113,7 +150,7 @@ public final class XRYFileReader implements AutoCloseable {
return true;
}
} else {
xryEntity.append(line).append("\n");
xryEntity.append(line).append('\n');
}
}
@@ -123,6 +160,7 @@ public final class XRYFileReader implements AutoCloseable {
/**
* Returns an XRY entity if there is one, otherwise an exception is thrown.
* Clients should test for another entity by calling hasNextEntity().
*
* @return A non-empty XRY entity.
* @throws IOException if an I/O error occurs.
@@ -139,6 +177,23 @@ public final class XRYFileReader implements AutoCloseable {
}
}
/**
* Peek at the next XRY entity without consuming it. If there are not more
* XRY entities left, an exception is thrown. Clients should test for
* another entity by calling hasNextEntity().
*
* @return A non-empty XRY entity.
* @throws IOException if an I/O error occurs.
* @throws NoSuchElementException if there are no more XRY entities to peek.
*/
public String peek() throws IOException {
if (hasNextEntity()) {
return xryEntity.toString();
} else {
throw new NoSuchElementException();
}
}
/**
* Closes any file handles this reader may have open.
*

View File

@@ -30,7 +30,7 @@ import java.util.stream.Stream;
/**
* Extracts XRY files and (optionally) non-XRY files from a XRY (Report) folder.
*/
public final class XRYFolder {
final class XRYFolder {
//Depth that will contain XRY files. All XRY files will be immediate
//children of their parent folder.

View File

@@ -0,0 +1,503 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.io.IOException;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Parses Messages-SMS files and creates artifacts.
*/
final class XRYMessagesFileParser implements XRYFileParser {
private static final Logger logger = Logger.getLogger(
XRYMessagesFileParser.class.getName());
private static final String PARSER_NAME = "XRY DSP";
private static final char KEY_VALUE_DELIMITER = ':';
private static final DateTimeFormatter DATE_TIME_PARSER
= DateTimeFormatter.ofPattern("M/d/y h:m:s [a][ z]");
//Meta keys. These describe how the XRY message entites are split
//up in the report file.
private static final String SEGMENT_COUNT = "segments";
private static final String SEGMENT_NUMBER = "segment number";
private static final String REFERENCE_NUMBER = "reference number";
//A more readable version of these values. Referring to if the user
//has read the message.
private static final int READ = 1;
private static final int UNREAD = 0;
private static final String TEXT_KEY = "text";
//All known XRY keys for message reports.
private static final Set<String> XRY_KEYS = new HashSet<String>() {
{
add(TEXT_KEY);
add("direction");
add("time");
add("status");
add("tel");
add("storage");
add("index");
add("folder");
add("service center");
add("type");
add("name");
}
};
//All known XRY namespaces for message reports.
private static final Set<String> XRY_NAMESPACES = new HashSet<String>() {
{
add("to");
add("from");
add("participant");
}
};
//All known meta keys.
private static final Set<String> XRY_META_KEYS = new HashSet<String>() {
{
add(REFERENCE_NUMBER);
add(SEGMENT_NUMBER);
add(SEGMENT_COUNT);
}
};
/**
* Message-SMS report artifacts can span multiple XRY entities and their
* attributes can span multiple lines. The "Text" key is the only known key
* value pair that can span multiple lines. Messages can be segmented,
* meaning that their "Text" content can appear in multiple XRY entities.
* Our goal for a segmented message is to aggregate all of the text pieces and
* create 1 artifact.
*
* This parse implementation assumes that segments are contiguous and that
* they ascend incrementally. There are checks in place to verify this
* assumption is correct, otherwise an error will appear in the logs.
*
* @param reader The XRYFileReader that reads XRY entities from the
* Message-SMS report.
* @param parent The parent Content to create artifacts from.
* @throws IOException If an I/O error is encountered during report reading
* @throws TskCoreException If an error during artifact creation is
* encountered.
*/
@Override
public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException {
Path reportPath = reader.getReportPath();
logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString()));
//Keep track of the reference numbers that have been parsed.
Set<Integer> referenceNumbersSeen = new HashSet<>();
while (reader.hasNextEntity()) {
String xryEntity = reader.nextEntity();
String[] xryLines = xryEntity.split("\n");
//First line of the entity is the title.
if (xryLines.length > 0) {
logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
}
List<BlackboardAttribute> attributes = new ArrayList<>();
String namespace = "";
for (int i = 1; i < xryLines.length; i++) {
String xryLine = xryLines[i];
String normalizedXryLine = xryLine.trim().toLowerCase();
if (XRY_NAMESPACES.contains(normalizedXryLine)) {
namespace = xryLine.trim();
continue;
}
//Find the XRY key on this line.
int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER);
if (keyDelimiter == -1) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value "
+ "pair on this line (in brackets) [ %s ], but one was not detected."
+ " Is this the continuation of a previous line?"
+ " Here is the previous line (in brackets) [ %s ]. "
+ "What does this key mean?", xryLine, xryLines[i - 1]));
continue;
}
//Extract the key value pair
String key = xryLine.substring(0, keyDelimiter).trim();
String value = xryLine.substring(keyDelimiter + 1).trim();
String normalizedKey = key.toLowerCase();
if (XRY_META_KEYS.contains(normalizedKey)) {
//Skip meta keys, they are being dealt with seperately.
continue;
}
if (!XRY_KEYS.contains(normalizedKey)) {
logger.log(Level.SEVERE, String.format("[XRY DSP] The following key, "
+ "value pair (in brackets, respectively) [ %s ], [ %s ] "
+ "was not recognized. Discarding... Here is the previous line "
+ "[ %s ] for context. What does this key mean?", key, value, xryLines[i - 1]));
continue;
}
if (value.isEmpty()) {
logger.log(Level.SEVERE, String.format("[XRY DSP] The following key "
+ "(in brackets) [ %s ] was recognized, but the value "
+ "was empty. Discarding... Here is the previous line "
+ "for context [ %s ]. Is this a continuation of this line? "
+ "What does an empty key mean?", key, xryLines[i - 1]));
continue;
}
//Assume text is the only field that can span multiple lines.
if (normalizedKey.equals(TEXT_KEY)) {
//Build up multiple lines.
for (; (i + 1) < xryLines.length
&& !hasKey(xryLines[i + 1])
&& !hasNamespace(xryLines[i + 1]); i++) {
String continuedValue = xryLines[i + 1].trim();
//Assume multi lined values are split by word.
value = value + " " + continuedValue;
}
int referenceNumber = getMetaInfo(xryLines, REFERENCE_NUMBER);
//Check if there is any segmented text. Min val is used to
//signify that no reference number was found.
if (referenceNumber != Integer.MIN_VALUE) {
logger.log(Level.INFO, String.format("[XRY DSP] Message entity "
+ "appears to be segmented with reference number [ %d ]", referenceNumber));
if (referenceNumbersSeen.contains(referenceNumber)) {
logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already "
+ "been seen. This means that the segments are not "
+ "contiguous. Any segments contiguous with this "
+ "one will be aggregated and another "
+ "(otherwise duplicate) artifact will be created.", referenceNumber));
}
referenceNumbersSeen.add(referenceNumber);
int segmentNumber = getMetaInfo(xryLines, SEGMENT_NUMBER);
//Unify segmented text, if there is any.
String segmentedText = getSegmentedText(referenceNumber,
segmentNumber, reader);
//Assume it was segmented by word.
value = value + " " + segmentedText;
}
}
BlackboardAttribute attribute = makeAttribute(namespace, key, value);
if (attribute != null) {
attributes.add(attribute);
}
}
//Only create artifacts with non-empty attributes.
if(!attributes.isEmpty()) {
BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE);
artifact.addAttributes(attributes);
}
}
}
/**
* Builds up segmented message entities so that the text is unified in the
* artifact.
*
* @param referenceNumber Reference number that messages are group by
* @param segmentNumber Segment number of the starting segment.
* @param reader
* @return
* @throws IOException
*/
private String getSegmentedText(int referenceNumber, int segmentNumber, XRYFileReader reader) throws IOException {
StringBuilder segmentedText = new StringBuilder();
int currentSegmentNumber = segmentNumber;
while (reader.hasNextEntity()) {
//Peek at the next to see if it has the same reference number.
String nextEntity = reader.peek();
String[] nextEntityLines = nextEntity.split("\n");
int nextReferenceNumber = getMetaInfo(nextEntityLines, REFERENCE_NUMBER);
if (nextReferenceNumber != referenceNumber) {
//Don't consume the next entity. It is not related
//to the current message thread.
break;
}
//Consume the entity.
reader.nextEntity();
int nextSegmentNumber = getMetaInfo(nextEntityLines, SEGMENT_NUMBER);
//Extract the text key from the entity, which is potentially
//multi-lined.
if (nextEntityLines.length > 0) {
logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] "
+ "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber));
}
if (nextSegmentNumber != currentSegmentNumber + 1) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous "
+ "segments are not ascending incrementally. Encountered "
+ "segment [ %d ] after segment [ %d ]. This means the reconstructed "
+ "text will be out of order.", nextSegmentNumber, currentSegmentNumber));
}
for (int i = 1; i < nextEntityLines.length; i++) {
String xryLine = nextEntityLines[i];
//Find the XRY key on this line.
int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER);
if (keyDelimiter == -1) {
//Skip this line, we are searching only for a text key-value pair.
continue;
}
String key = xryLine.substring(0, keyDelimiter);
String normalizedKey = key.trim().toLowerCase();
if (normalizedKey.equals(TEXT_KEY)) {
String value = xryLine.substring(keyDelimiter + 1).trim();
segmentedText.append(value).append(' ');
//Build up multiple lines.
for (; (i + 1) < nextEntityLines.length
&& !hasKey(nextEntityLines[i + 1])
&& !hasNamespace(nextEntityLines[i + 1]); i++) {
String continuedValue = nextEntityLines[i + 1].trim();
segmentedText.append(continuedValue).append(' ');
}
}
}
currentSegmentNumber = nextSegmentNumber;
}
//Remove the trailing space.
if (segmentedText.length() > 0) {
segmentedText.setLength(segmentedText.length() - 1);
}
return segmentedText.toString();
}
/**
* Determines if the line has recognized key value on it.
*
* @param xryLine
* @return
*/
private boolean hasKey(String xryLine) {
int delimiter = xryLine.indexOf(':');
if (delimiter != -1) {
String key = xryLine.substring(0, delimiter);
String normalizedKey = key.trim().toLowerCase();
return XRY_KEYS.contains(normalizedKey);
} else {
return false;
}
}
/**
* Determines if the line is a recognized namespace.
*
* @param xryLine
* @return
*/
private boolean hasNamespace(String xryLine) {
String normalizedLine = xryLine.trim().toLowerCase();
return XRY_NAMESPACES.contains(normalizedLine);
}
/**
* Extracts meta keys from the XRY entity. All of the known meta
* keys are integers and describe the message segments.
*
* @param xryLines Current XRY entity
* @param expectedKey The meta key to search for
* @return The interpreted integer value or Integer.MIN_VALUE if
* no meta key was found.
*/
private int getMetaInfo(String[] xryLines, String metaKey) {
for (int i = 0; i < xryLines.length; i++) {
String xryLine = xryLines[i];
String normalizedXryLine = xryLine.trim().toLowerCase();
int firstDelimiter = normalizedXryLine.indexOf(KEY_VALUE_DELIMITER);
if (firstDelimiter != -1) {
String key = normalizedXryLine.substring(0, firstDelimiter);
if (key.equals(metaKey)) {
String value = normalizedXryLine.substring(firstDelimiter + 1).trim();
try {
return Integer.parseInt(value);
} catch (NumberFormatException ex) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for "
+ "meta key [ %s ] was not an integer.", value, metaKey), ex);
}
}
}
}
return Integer.MIN_VALUE;
}
/**
* Creates an attribute from the extracted key value pair.
*
* @param nameSpace The namespace of this key value pair.
* It will have been verified beforehand, otherwise it will be empty.
* @param key The key that was verified beforehand
* @param value The value associated with that key.
* @return
*/
private BlackboardAttribute makeAttribute(String namespace, String key, String value) {
String normalizedKey = key.toLowerCase();
String normalizedNamespace = namespace.toLowerCase();
String normalizedValue = value.toLowerCase();
switch (normalizedKey) {
case "time":
//Tranform value to epoch ms
try {
String dateTime = removeDateTimeLocale(value);
String normalizedDateTime = dateTime.trim();
long dateTimeInEpoch = calculateSecondsSinceEpoch(normalizedDateTime);
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, PARSER_NAME, dateTimeInEpoch);
} catch (DateTimeParseException ex) {
logger.log(Level.SEVERE, String.format("[XRY DSP] Assumption "
+ "about the date time formatting of messages is not "
+ "right. Here is the value [ %s ].", value), ex);
return null;
}
case "direction":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, PARSER_NAME, value);
case "text":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT, PARSER_NAME, value);
case "status":
switch (normalizedValue) {
case "read":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, READ);
case "unread":
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, UNREAD);
case "sending failed":
//Ignore for now.
return null;
case "deleted":
//Ignore for now.
return null;
case "unsent":
//Ignore for now.
return null;
default:
logger.log(Level.SEVERE, String.format("[XRY DSP] Unrecognized "
+ "status value [ %s ].", value));
return null;
}
case "type":
switch (normalizedValue) {
case "deliver":
//Ignore for now.
return null;
case "submit":
//Ignore for now.
return null;
case "status report":
//Ignore for now.
return null;
default:
logger.log(Level.SEVERE, String.format("[XRY DSP] Unrecognized "
+ "type value [ %s ]", value));
return null;
}
case "storage":
//Ignore for now.
return null;
case "index":
//Ignore for now.
return null;
case "folder":
//Ignore for now.
return null;
case "name":
//Ignore for now.
return null;
case "service center":
//Ignore for now.
return null;
case "tel":
//Apply the namespace
if (normalizedNamespace.equals("from")) {
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, PARSER_NAME, value);
} else {
//Assume to and participant are both equivalent to TSK_PHONE_NUMBER_TO
return new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, PARSER_NAME, value);
}
default:
throw new IllegalArgumentException(String.format("key [ %s ] was not recognized.", key));
}
}
/**
* Removes the locale from the date time value.
*
* Locale in this case being (Device) or (Network).
*
* @param dateTime XRY datetime value to be sanitized.
* @return A purer date time value.
*/
private String removeDateTimeLocale(String dateTime) {
int index = dateTime.indexOf('(');
if (index == -1) {
return dateTime;
}
return dateTime.substring(0, index);
}
/**
* Parses the date time value and calculates ms since epoch. The time zone is
* assumed to be UTC.
*
* @param dateTime
* @return
*/
private long calculateSecondsSinceEpoch(String dateTime) {
LocalDateTime localDateTime = LocalDateTime.parse(dateTime, DATE_TIME_PARSER);
//Assume dates have no offset.
return localDateTime.toInstant(ZoneOffset.UTC).getEpochSecond();
}
}

View File

@@ -0,0 +1,66 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.util.Map;
import java.util.HashMap;
import java.util.List;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Parses XRY Web-Bookmark files and creates artifacts.
*/
final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser {
//All known XRY keys for web bookmarks.
private static final Map<String, BlackboardAttribute.ATTRIBUTE_TYPE> KEY_TO_TYPE
= new HashMap<String, BlackboardAttribute.ATTRIBUTE_TYPE>() {
{
put("web address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL);
put("domain", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN);
}
};
@Override
boolean isKey(String key) {
String normalizedKey = key.toLowerCase();
return KEY_TO_TYPE.containsKey(normalizedKey);
}
@Override
boolean isNamespace(String nameSpace) {
//No known namespaces for web reports.
return false;
}
@Override
BlackboardAttribute makeAttribute(String nameSpace, String key, String value) {
String normalizedKey = key.toLowerCase();
return new BlackboardAttribute(KEY_TO_TYPE.get(normalizedKey), PARSER_NAME, value);
}
@Override
void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException {
BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
artifact.addAttributes(attributes);
}
}