From 422ff6330ec5b5c30d634d70d960c20b6af71b0e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 9 Sep 2019 13:48:09 -0400 Subject: [PATCH 01/55] Moved work to use this branch as base --- .../android/ResultSetIterator.py | 35 +++ .../android/TskCallLogsParser.py | 59 +++++ .../android/TskContactsParser.py | 49 ++++ .../android/TskMessagesParser.py | 68 ++++++ InternalPythonModules/android/line.py | 225 ++++++++++++++++++ InternalPythonModules/android/module.py | 3 +- 6 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py create mode 100644 InternalPythonModules/android/line.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..77c7aa12da --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,59 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self.INCOMING_MSG_STRING = "Incoming" + self.OUTGOING_MSG_STRING = "Outgoing" + self._DEFAULT_STRING = "" + self._DEFAULT_LONG = -1L + + def get_account_name(self): + return self._DEFAULT_STRING + + def get_call_direction(self): + return self._DEFAULT_STRING + + def get_phone_number_from(self): + return self._DEFAULT_STRING + + def get_phone_number_to(self): + return self._DEFAULT_STRING + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_contact_name(self): + return self._DEFAULT_STRING diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..0346a203e7 --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,68 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self.INCOMING_MSG_STRING = "Incoming" + self.OUTGOING_MSG_STRING = "Outgoing" + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_INT = -1 + + def get_account_id(self): + return self._DEFAULT_TEXT + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_TEXT + + def get_phone_number_from(self): + return self._DEFAULT_TEXT + + def get_phone_number_to(self): + return self._DEFAULT_TEXT + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_INT + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py new file mode 100644 index 0000000000..67589fcbea --- /dev/null +++ b/InternalPythonModules/android/line.py @@ -0,0 +1,225 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from java.util import ArrayList +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.datamodel import ContentUtils +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskContactsParser import TskContactsParser +from TskMessagesParser import TskMessagesParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class LineAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the Line App databases for TSK contacts & message artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._LINE_PACKAGE_NAME = "jp.naver.line.android" + self._PARSER_NAME = "Line Parser" + + def analyze(self, dataSource, fileManager, context): + try: + contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line", self._LINE_PACKAGE_NAME) + calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", self._LINE_PACKAGE_NAME) + + for contact_and_message_db in contact_and_message_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) + + contacts_parser = LineContactsParser(contact_and_message_db) + while contacts_parser.next(): + blackboard_util.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + """ + messages_parser = LineMessagesParser(line_db) + while messages_parser.next(): + blackboard_util.addMessage( + messages_parser.get_account_id(), + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + """ + contact_and_message_db.close() + + for calllog_db in calllog_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) + + calllog_parser = LineCallLogsParser(calllog_db) + while calllog_parser.next(): + blackboard_util.addCalllog( + calllog_parser.get_account_name(), + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_contact_name() + ) + + calllog_parser.close() + calllog_db.close() + + except (SQLException, TskCoreException) as ex: + # Error parsing Line databases. + self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + +class LineCallLogsParser(TskCallLogsParser): + """ + Parses out TSK_CALLLOG information from the Line database. + TSK_CALLLOG fields that are not in the line database are given + a default value inherited from the super class. + """ + + def __init__(self, calllog_db): + super(LineCallLogsParser, self).__init__(calllog_db.runQuery( + """ + SELECT C.caller_mid AS mid, + substr(C.call_type, -1) AS direction, + C.start_time AS start_time, + C.end_time AS end_time + FROM call_history AS C + """ + ) + ) + self._OUTGOING_CALL = "O" + self._INCOMING_CALL = "I" + self._had_error = False + + def get_call_direction(self): + direction = self.result_set.getString("direction") + if direction == self._OUTGOING_CALL: + return self.OUTGOING_MSG_STRING + return self.INCOMING_MSG_STRING + + def get_call_start_date_time(self): + start_time = self.result_set.getString("start_time") + try: + return long(start_time) / 1000 + except ValueError as ve: + self._had_error = True + + def get_call_end_date_time(self): + end_time = self.result_set.getString("end_time") + try: + return long(end_time) / 1000 + except ValueError as ve: + self._had_error = True + +class LineContactsParser(TskContactsParser): + """ + Parses out TSK_CONTACT information from the Line database. + TSK_CONTACT fields that are not in the line database are given + a default value inherited from the super class. + """ + + def __init__(self, contact_db): + super(LineContactsParser, self).__init__(contact_db.runQuery( + """ + SELECT name, + server_name + FROM contacts + """ + ) + ) + def get_account_name(self): + return self.result_set.getString("server_name") + + def get_contact_name(self): + return self.result_set.getString("name") + +class LineMessagesParser(TskMessagesParser): + """ + Parse out TSK_MESSAGE information from the Line database. + TSK_MESSAGE fields that are not in the line database are given + a default value inherited from the super class. + """ + + def __init__(self, message_db): + super().__init__(message_db.runQuery( + """SELECT created_time, content, contacts.server_name AS server_name, read_count + FROM chat_history + JOIN contacts + ON chat_history.from_mid = contacts.m_id""" + )) + self._LINE_MESSAGE_TYPE = "Line Message" + self._had_error = False + + def get_account_id(self): + return self.result_set.getString("server_name") + + def get_message_type(self): + return self.LINE_MESSAGE_TYPE + + def get_phone_number_from(self): + return self.result_set("server_name") + + def get_message_date_time(self): + created_time = self.result_set.getString("created_time") + try: + #Get time in seconds (created_time is stored in ms from epoch) + return long(created_time) / 1000 + except ValueError as ve: + self._had_error = True + return super(LineMessagesParser, self).get_message_date_time() + + def get_message_text(self): + content = self.result_set.getString("content") + if not LineContentUtil.is_text_message(content): + return "" + return content diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..9893df2b74 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import line class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), line.LineAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) From b773f8a03db96dbfe7e0dc1e7d06e748031d43f4 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 9 Sep 2019 16:37:21 -0400 Subject: [PATCH 02/55] debugging line queries --- InternalPythonModules/android/line.py | 36 +++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index 67589fcbea..df8db5d088 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -60,8 +60,8 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): try: - contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line", self._LINE_PACKAGE_NAME) - calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", self._LINE_PACKAGE_NAME) + contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line.db", True, self._LINE_PACKAGE_NAME) + calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", True, self._LINE_PACKAGE_NAME) for contact_and_message_db in contact_and_message_dbs: blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) @@ -98,9 +98,17 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): for calllog_db in calllog_dbs: blackboard_util = BlackboardUtil(self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) + calllog_db.attachDatabase(dataSource, "naver_line.db", True, calllog_db.getDBFile().getParentPath(), "naver") calllog_parser = LineCallLogsParser(calllog_db) while calllog_parser.next(): + print(calllog_parser.get_account_name()) + print(calllog_parser.get_contact_name()) + print(calllog_parser.get_call_direction()) + print(calllog_parser.get_phone_number_from()) + print(calllog_parser.get_phone_number_to()) + print(calllog_parser.get_call_start_date_time()) + print(calllog_parser.get_call_end_date_time()) blackboard_util.addCalllog( calllog_parser.get_account_name(), calllog_parser.get_call_direction(), @@ -111,6 +119,7 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): calllog_parser.get_contact_name() ) + calllog_db.detachDatabase("naver") calllog_parser.close() calllog_db.close() @@ -129,11 +138,14 @@ class LineCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(LineCallLogsParser, self).__init__(calllog_db.runQuery( """ - SELECT C.caller_mid AS mid, - substr(C.call_type, -1) AS direction, - C.start_time AS start_time, - C.end_time AS end_time - FROM call_history AS C + SELECT substr(CallH.call_type, -1) AS direction, + CallH.start_time AS start_time, + CallH.end_time AS end_time, + ConT.server_name AS account_name, + ConT.name AS contact_name + FROM call_history AS CallH + JOIN naver.contacts AS ConT + ON CallH.caller_mid = ConT.m_id """ ) ) @@ -153,6 +165,8 @@ class LineCallLogsParser(TskCallLogsParser): return long(start_time) / 1000 except ValueError as ve: self._had_error = True + print("bad_conversion") + return super(LineCallLogsParser, self).get_call_start_date_time() def get_call_end_date_time(self): end_time = self.result_set.getString("end_time") @@ -160,6 +174,14 @@ class LineCallLogsParser(TskCallLogsParser): return long(end_time) / 1000 except ValueError as ve: self._had_error = True + print("bad conversion") + return super(LineCallLogsParser, self).get_call_end_date_time() + + def get_account_name(self): + return self.result_set.getString("account_name") + + def get_contact_name(self): + return self.result_set.getString("contact_name") class LineContactsParser(TskContactsParser): """ From 19bb7065ea90a02bdc0b1efe40b74a116096fcca Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 10 Sep 2019 14:27:16 -0400 Subject: [PATCH 03/55] Moved work to new branch --- .../android/ResultSetIterator.py | 35 +++ .../android/TskCallLogsParser.py | 58 ++++ .../android/TskContactsParser.py | 49 ++++ .../android/TskMessagesParser.py | 68 +++++ InternalPythonModules/android/viber.py | 274 ++++++++++++++++++ 5 files changed, 484 insertions(+) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py create mode 100644 InternalPythonModules/android/viber.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..22b509d612 --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,58 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self.INCOMING_MSG_STRING = "Incoming" + self.OUTGOING_MSG_STRING = "Outgoing" + self._DEFAULT_STRING = "" + + def get_account_name(self): + return self._DEFAULT_STRING + + def get_call_direction(self): + return self._DEFAULT_STRING + + def get_phone_number_from(self): + return self._DEFAULT_STRING + + def get_phone_number_to(self): + return self._DEFAULT_STRING + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_contact_name(self): + return self._DEFAULT_STRING diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..0346a203e7 --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,68 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self.INCOMING_MSG_STRING = "Incoming" + self.OUTGOING_MSG_STRING = "Outgoing" + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_INT = -1 + + def get_account_id(self): + return self._DEFAULT_TEXT + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_TEXT + + def get_phone_number_from(self): + return self._DEFAULT_TEXT + + def get_phone_number_to(self): + return self._DEFAULT_TEXT + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_INT + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py new file mode 100644 index 0000000000..3a65382c3c --- /dev/null +++ b/InternalPythonModules/android/viber.py @@ -0,0 +1,274 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskMessagesParser import TskMessagesParser +from TskContactsParser import TskContactsParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class ViberAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the Viber App databases for TSK contacts, message and calllog artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._VIBER_PACKAGE_NAME = "com.viber.voip" + self._PARSER_NAME = "Viber Parser" + + def analyze(self, dataSource, fileManager, context): + """ + Extract, Transform and Load all messages, contacts and calllogs from the Viber databases. + """ + + try: + contact_and_calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_data", self._VIBER_PACKAGE_NAME) + message_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_messages", self._VIBER_PACKAGE_NAME) + + #Extract TSK_CONTACT and TSK_CALLLOG information + for contact_and_calllog_db in contact_and_calllog_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_calllog_db.getDBFile(), Account.Type.VIBER) + + contacts_parser = ViberContactsParser(contact_and_calllog_db) + while contacts_parser.next(): + blackboard_util.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + calllog_parser = ViberCallLogsParser(contact_and_calllog_db) + while calllog_parser.next(): + blackboard_util.addCalllog( + calllog_parser.get_account_name(), + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_contact_name() + ) + calllog_parser.close() + + #Extract TSK_MESSAGE information + for message_db in message_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, message_db.getDBFile(), Account.Type.VIBER) + messages_parser = ViberMessagesParser(message_db) + while messages_parser.next(): + blackboard_util.addMessage( + messages_parser.get_account_id(), + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except (SQLException, TskCoreException) as ex: + #Error parsing Viber db + self._logger.log(Level.WARNING, "Error parsing Viber Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + +class ViberCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from the Viber database. + TSK_CALLLOG fields that are not in the Viber database are given + a default value inherited from the super class. + """ + + def __init__(self, calllog_db): + super(ViberCallLogsParser, self).__init__(calllog_db.runQuery( + """ + SELECT C.canonized_number AS number, + C.type AS direction, + C.duration AS seconds, + C.date AS start_time + FROM calls AS C + """ + ) + ) + + self._OUTGOING_CALL = 2 + self._INCOMING_CALL = 1 + self._MISSED_CALL = 3 + + def get_phone_number_from(self): + if self.get_call_direction() == self.INCOMING_MSG_STRING: + return self.result_set.getString("number") + #Give default value if the call is outgoing, the device's # is not stored in the database. + return super(ViberCallLogsParser, self).get_phone_number_from() + + def get_phone_number_to(self): + if self.get_call_direction() == self.OUTGOING_MSG_STRING: + return self.result_set.getString("number") + #Give default value if the call is incoming, the device's # is not stored in the database. + return super(ViberCallLogsParser, self).get_phone_number_to() + + def get_call_direction(self): + direction = self.result_set.getInt("direction") + if direction == self._INCOMING_CALL or direction == self._MISSED_CALL: + return self.INCOMING_MSG_STRING + return self.OUTGOING_MSG_STRING + + def get_call_start_date_time(self): + return self.result_set.getLong("start_time") / 1000 + + def get_call_end_date_time(self): + start_time = self.get_call_start_date_time() + duration = self.result_set.getLong("seconds") + return start_time + duration + +class ViberContactsParser(TskContactsParser): + """ + Extracts TSK_CONTACT information from the Viber database. + TSK_CONTACT fields that are not in the Viber database are given a default value + inherited from the super class. + """ + + def __init__(self, contact_db): + super(ViberContactsParser, self).__init__(contact_db.runQuery( + """ + SELECT C.display_name AS name, + D.data2 AS number + FROM phonebookcontact AS C + JOIN phonebookdata AS D + ON C._id = D.contact_id + """ + ) + ) + + def get_account_name(self): + return self.result_set.getString("name") + + def get_contact_name(self): + return self.get_account_name() + + def get_phone(self): + return self.result_set.getString("number") + +class ViberMessagesParser(TskMessagesParser): + """ + Extract TSK_MESSAGE information from the Viber database. + TSK_CONTACT fields that are not in the Viber database are given a default value + inherited from the super class. + """ + + def __init__(self, message_db): + super(ViberMessagesParser, self).__init__(message_db.runQuery( + """ + SELECT FROM_RESULT.number AS from_number, + FROM_RESULT.viber_name AS from_name, + TO_RESULT.number AS to_number, + M.conversation_id AS thread_id, + M.body AS msg_content, + M.send_type AS direction, + M.msg_date AS msg_date, + M.unread AS read_status + FROM (SELECT P._id, + P.conversation_id, + PI.number, + PI.viber_name + FROM participants AS P + JOIN participants_info AS PI + ON P.participant_info_id = PI._id) AS FROM_RESULT + JOIN (SELECT P._id, + P.conversation_id, + PI.number + FROM participants AS P + JOIN participants_info AS PI + ON P.participant_info_id = PI._id) AS TO_RESULT + ON FROM_RESULT._id != TO_RESULT._id + AND FROM_RESULT.conversation_id = TO_RESULT.conversation_id + JOIN messages AS M + ON M.participant_id = FROM_RESULT._id + AND M.conversation_id = FROM_RESULT.conversation_id + """ + ) + ) + self._VIBER_MESSAGE_TYPE = "Viber Message" + self._INCOMING_MESSAGE_TYPE = 0 + self._OUTGOING_MESSAGE_TYPE = 1 + + def get_account_id(self): + name = self.result_set.getString("from_name") + if name is None or len(name) == 0: + return self.get_phone_number_from() + return name + + def get_message_type(self): + return self._VIBER_MESSAGE_TYPE + + def get_phone_number_from(self): + return self.result_set.getString("from_number") + + def get_message_direction(self): + direction = self.result_set.getInt("direction") + if direction == self._INCOMING_MESSAGE_TYPE: + return self.INCOMING_MSG_STRING + return self.OUTGOING_MSG_STRING + + def get_phone_number_to(self): + return self.result_set.getString("to_number") + + def get_message_date_time(self): + #transform from ms to seconds + return self.result_set.getLong("msg_date") / 1000 + + def get_message_read_status(self): + #Viber: 0 is read, 1 is unread. + #TSK_MESSAGE 1 is read, 0 is unread. + if self.get_message_direction() == self.INCOMING_MSG_STRING: + return 1 - self.result_set.getInt("read_status") + return super(ViberMessagesParser, self).get_message_read_status() + + def get_message_text(self): + return self.result_set.getString("msg_content") + + def get_thread_id(self): + return str(self.result_set.getInt("thread_id")) From 22968498b8942aae7e28bd62c759febb7045c4e0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 10 Sep 2019 18:16:24 -0400 Subject: [PATCH 04/55] Finished viber with new API changes --- .../android/TskMessagesParser.py | 18 ++-- InternalPythonModules/android/module.py | 3 +- InternalPythonModules/android/viber.py | 94 ++++++++++--------- 3 files changed, 60 insertions(+), 55 deletions(-) diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 0346a203e7..e3edbb25c8 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper class TskMessagesParser(ResultSetIterator): """ @@ -31,14 +33,12 @@ class TskMessagesParser(ResultSetIterator): def __init__(self, result_set): super(TskMessagesParser, self).__init__(result_set) - self.INCOMING_MSG_STRING = "Incoming" - self.OUTGOING_MSG_STRING = "Outgoing" + self.INCOMING_MSG = "Incoming" + self.OUTGOING_MSG = "Outgoing" self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_INT = -1 - - def get_account_id(self): - return self._DEFAULT_TEXT + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") def get_message_type(self): return self._DEFAULT_TEXT @@ -47,16 +47,16 @@ class TskMessagesParser(ResultSetIterator): return self._DEFAULT_TEXT def get_phone_number_from(self): - return self._DEFAULT_TEXT + return self._DEFAULT_ACCOUNT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_TEXT + return self._DEFAULT_ACCOUNT_ADDRESS def get_message_date_time(self): return self._DEFAULT_LONG def get_message_read_status(self): - return self._DEFAULT_INT + return self._DEFAULT_MSG_READ_STATUS def get_message_subject(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..b522704bac 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import viber class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), viber.ViberAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 3a65382c3c..77a4560839 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -61,8 +61,8 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): """ try: - contact_and_calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_data", self._VIBER_PACKAGE_NAME) - message_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_messages", self._VIBER_PACKAGE_NAME) + contact_and_calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_data", True, self._VIBER_PACKAGE_NAME) + message_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_messages", True, self._VIBER_PACKAGE_NAME) #Extract TSK_CONTACT and TSK_CALLLOG information for contact_and_calllog_db in contact_and_calllog_dbs: @@ -91,6 +91,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): calllog_parser.get_contact_name() ) calllog_parser.close() + contact_and_calllog_db.close() #Extract TSK_MESSAGE information for message_db in message_dbs: @@ -98,7 +99,6 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): messages_parser = ViberMessagesParser(message_db) while messages_parser.next(): blackboard_util.addMessage( - messages_parser.get_account_id(), messages_parser.get_message_type(), messages_parser.get_message_direction(), messages_parser.get_phone_number_from(), @@ -110,6 +110,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() + message_db.close() except (SQLException, TskCoreException) as ex: #Error parsing Viber db self._logger.log(Level.WARNING, "Error parsing Viber Databases", ex) @@ -138,6 +139,9 @@ class ViberCallLogsParser(TskCallLogsParser): self._INCOMING_CALL = 1 self._MISSED_CALL = 3 + def get_account_name(self): + return self.result_set.getString("number") + def get_phone_number_from(self): if self.get_call_direction() == self.INCOMING_MSG_STRING: return self.result_set.getString("number") @@ -184,10 +188,10 @@ class ViberContactsParser(TskContactsParser): ) def get_account_name(self): - return self.result_set.getString("name") + return self.result_set.getString("number") def get_contact_name(self): - return self.get_account_name() + return self.result_set.getString("name") def get_phone(self): return self.result_set.getString("number") @@ -202,32 +206,33 @@ class ViberMessagesParser(TskMessagesParser): def __init__(self, message_db): super(ViberMessagesParser, self).__init__(message_db.runQuery( """ - SELECT FROM_RESULT.number AS from_number, - FROM_RESULT.viber_name AS from_name, - TO_RESULT.number AS to_number, - M.conversation_id AS thread_id, - M.body AS msg_content, - M.send_type AS direction, - M.msg_date AS msg_date, - M.unread AS read_status - FROM (SELECT P._id, - P.conversation_id, - PI.number, - PI.viber_name - FROM participants AS P - JOIN participants_info AS PI - ON P.participant_info_id = PI._id) AS FROM_RESULT - JOIN (SELECT P._id, - P.conversation_id, - PI.number - FROM participants AS P - JOIN participants_info AS PI - ON P.participant_info_id = PI._id) AS TO_RESULT - ON FROM_RESULT._id != TO_RESULT._id - AND FROM_RESULT.conversation_id = TO_RESULT.conversation_id - JOIN messages AS M - ON M.participant_id = FROM_RESULT._id - AND M.conversation_id = FROM_RESULT.conversation_id + SELECT convo_participants.from_number AS from_number, + convo_participants.recipients AS recipients, + M.conversation_id AS thread_id, + M.body AS msg_content, + M.send_type AS direction, + M.msg_date AS msg_date, + M.unread AS read_status + FROM (SELECT *, + group_concat(TO_RESULT.number) AS recipients + FROM (SELECT P._id AS FROM_ID, + P.conversation_id, + PI.number AS FROM_NUMBER + FROM participants AS P + JOIN participants_info AS PI + ON P.participant_info_id = PI._id) AS FROM_RESULT + JOIN (SELECT P._id AS TO_ID, + P.conversation_id, + PI.number + FROM participants AS P + JOIN participants_info AS PI + ON P.participant_info_id = PI._id) AS TO_RESULT + ON FROM_RESULT.from_id != TO_RESULT.to_id + AND FROM_RESULT.conversation_id = TO_RESULT.conversation_id + GROUP BY FROM_RESULT.from_id) AS convo_participants + JOIN messages AS M + ON M.participant_id = convo_participants.from_id + AND M.conversation_id = convo_participants.conversation_id """ ) ) @@ -235,36 +240,35 @@ class ViberMessagesParser(TskMessagesParser): self._INCOMING_MESSAGE_TYPE = 0 self._OUTGOING_MESSAGE_TYPE = 1 - def get_account_id(self): - name = self.result_set.getString("from_name") - if name is None or len(name) == 0: - return self.get_phone_number_from() - return name - def get_message_type(self): return self._VIBER_MESSAGE_TYPE def get_phone_number_from(self): - return self.result_set.getString("from_number") + return Account.Address(self.result_set.getString("from_number"), + self.result_set.getString("from_number")) def get_message_direction(self): direction = self.result_set.getInt("direction") if direction == self._INCOMING_MESSAGE_TYPE: - return self.INCOMING_MSG_STRING - return self.OUTGOING_MSG_STRING + return self.INCOMING_MSG + return self.OUTGOING_MSG def get_phone_number_to(self): - return self.result_set.getString("to_number") + recipients = [] + for token in self.result_set.getString("recipients").split(","): + recipients.append(Account.Address(token, token)) + return recipients def get_message_date_time(self): #transform from ms to seconds return self.result_set.getLong("msg_date") / 1000 def get_message_read_status(self): - #Viber: 0 is read, 1 is unread. - #TSK_MESSAGE 1 is read, 0 is unread. - if self.get_message_direction() == self.INCOMING_MSG_STRING: - return 1 - self.result_set.getInt("read_status") + if self.get_message_direction() == self.INCOMING_MSG: + if self.result_set.getInt("read_status") == 0: + return BlackboardUtil.MessageReadStatusEnum.READ + else: + return BlackboardUtil.MessageReadStatusEnum.UNREAD return super(ViberMessagesParser, self).get_message_read_status() def get_message_text(self): From c03377b658acd7ea37e681f6ea85d423eb2e59b6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 09:43:55 -0400 Subject: [PATCH 05/55] Refactoring/code clean up --- .../android/TskCallLogsParser.py | 4 ++-- InternalPythonModules/android/viber.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 22b509d612..66ea27eee3 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -32,8 +32,8 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) - self.INCOMING_MSG_STRING = "Incoming" - self.OUTGOING_MSG_STRING = "Outgoing" + self.INCOMING_CALL = "Incoming" + self.OUTGOING_CALL = "Outgoing" self._DEFAULT_STRING = "" def get_account_name(self): diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 77a4560839..1505b9956a 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -135,30 +135,30 @@ class ViberCallLogsParser(TskCallLogsParser): ) ) - self._OUTGOING_CALL = 2 - self._INCOMING_CALL = 1 - self._MISSED_CALL = 3 + self._OUTGOING_CALL_TYPE = 2 + self._INCOMING_CALL_TYPE = 1 + self._MISSED_CALL_TYPE = 3 def get_account_name(self): return self.result_set.getString("number") def get_phone_number_from(self): - if self.get_call_direction() == self.INCOMING_MSG_STRING: + if self.get_call_direction() == self.INCOMING_CALL: return self.result_set.getString("number") #Give default value if the call is outgoing, the device's # is not stored in the database. return super(ViberCallLogsParser, self).get_phone_number_from() def get_phone_number_to(self): - if self.get_call_direction() == self.OUTGOING_MSG_STRING: + if self.get_call_direction() == self.OUTGOING_CALL: return self.result_set.getString("number") #Give default value if the call is incoming, the device's # is not stored in the database. return super(ViberCallLogsParser, self).get_phone_number_to() def get_call_direction(self): direction = self.result_set.getInt("direction") - if direction == self._INCOMING_CALL or direction == self._MISSED_CALL: - return self.INCOMING_MSG_STRING - return self.OUTGOING_MSG_STRING + if direction == self._INCOMING_CALL_TYPE or direction == self._MISSED_CALL_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL def get_call_start_date_time(self): return self.result_set.getLong("start_time") / 1000 From 7ee98a04708f73a7a23aa2e53e732928fb5f2714 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 10:17:48 -0400 Subject: [PATCH 06/55] More code cleaup --- InternalPythonModules/android/viber.py | 50 ++++++++++++++++---------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 1505b9956a..64b9333432 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -29,8 +29,8 @@ from java.util.logging import Level from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger -from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact @@ -47,7 +47,8 @@ import general class ViberAnalyzer(general.AndroidComponentAnalyzer): """ - Parses the Viber App databases for TSK contacts, message and calllog artifacts. + Parses the Viber App databases for TSK contacts, message + and calllog artifacts. """ def __init__(self): @@ -57,20 +58,24 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): """ - Extract, Transform and Load all messages, contacts and calllogs from the Viber databases. + Extract, Transform and Load all messages, contacts and + calllogs from the Viber databases. """ try: - contact_and_calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_data", True, self._VIBER_PACKAGE_NAME) - message_dbs = SQLiteUtil.findAppDatabases(dataSource, "viber_messages", True, self._VIBER_PACKAGE_NAME) + contact_and_calllog_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "viber_data", True, self._VIBER_PACKAGE_NAME) + message_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "viber_messages", True, self._VIBER_PACKAGE_NAME) #Extract TSK_CONTACT and TSK_CALLLOG information for contact_and_calllog_db in contact_and_calllog_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_calllog_db.getDBFile(), Account.Type.VIBER) + parser_helper = AppDBParserHelper(self._PARSER_NAME, + contact_and_calllog_db.getDBFile(), Account.Type.VIBER) contacts_parser = ViberContactsParser(contact_and_calllog_db) while contacts_parser.next(): - blackboard_util.addContact( + parser_helper.addContact( contacts_parser.get_account_name(), contacts_parser.get_contact_name(), contacts_parser.get_phone(), @@ -79,9 +84,10 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): contacts_parser.get_email() ) contacts_parser.close() + calllog_parser = ViberCallLogsParser(contact_and_calllog_db) while calllog_parser.next(): - blackboard_util.addCalllog( + parser_helper.addCalllog( calllog_parser.get_account_name(), calllog_parser.get_call_direction(), calllog_parser.get_phone_number_from(), @@ -91,14 +97,17 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): calllog_parser.get_contact_name() ) calllog_parser.close() + contact_and_calllog_db.close() #Extract TSK_MESSAGE information for message_db in message_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, message_db.getDBFile(), Account.Type.VIBER) + parser_helper = AppDBParserHelper(self._PARSER_NAME, + message_db.getDBFile(), Account.Type.VIBER) + messages_parser = ViberMessagesParser(message_db) while messages_parser.next(): - blackboard_util.addMessage( + parser_helper.addMessage( messages_parser.get_message_type(), messages_parser.get_message_direction(), messages_parser.get_phone_number_from(), @@ -110,6 +119,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() + message_db.close() except (SQLException, TskCoreException) as ex: #Error parsing Viber db @@ -145,13 +155,15 @@ class ViberCallLogsParser(TskCallLogsParser): def get_phone_number_from(self): if self.get_call_direction() == self.INCOMING_CALL: return self.result_set.getString("number") - #Give default value if the call is outgoing, the device's # is not stored in the database. + #Give default value if the call is outgoing, + #the device's # is not stored in the database. return super(ViberCallLogsParser, self).get_phone_number_from() def get_phone_number_to(self): if self.get_call_direction() == self.OUTGOING_CALL: return self.result_set.getString("number") - #Give default value if the call is incoming, the device's # is not stored in the database. + #Give default value if the call is incoming, + #the device's # is not stored in the database. return super(ViberCallLogsParser, self).get_phone_number_to() def get_call_direction(self): @@ -171,8 +183,8 @@ class ViberCallLogsParser(TskCallLogsParser): class ViberContactsParser(TskContactsParser): """ Extracts TSK_CONTACT information from the Viber database. - TSK_CONTACT fields that are not in the Viber database are given a default value - inherited from the super class. + TSK_CONTACT fields that are not in the Viber database are given + a default value inherited from the super class. """ def __init__(self, contact_db): @@ -199,8 +211,8 @@ class ViberContactsParser(TskContactsParser): class ViberMessagesParser(TskMessagesParser): """ Extract TSK_MESSAGE information from the Viber database. - TSK_CONTACT fields that are not in the Viber database are given a default value - inherited from the super class. + TSK_CONTACT fields that are not in the Viber database are given + a default value inherited from the super class. """ def __init__(self, message_db): @@ -266,9 +278,9 @@ class ViberMessagesParser(TskMessagesParser): def get_message_read_status(self): if self.get_message_direction() == self.INCOMING_MSG: if self.result_set.getInt("read_status") == 0: - return BlackboardUtil.MessageReadStatusEnum.READ + return AppDBParserHelper.MessageReadStatusEnum.READ else: - return BlackboardUtil.MessageReadStatusEnum.UNREAD + return AppDBParserHelper.MessageReadStatusEnum.UNREAD return super(ViberMessagesParser, self).get_message_read_status() def get_message_text(self): From 97dc26f4d903123509316a3e31e03f0eed8e7c51 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 11:09:56 -0400 Subject: [PATCH 07/55] comments --- InternalPythonModules/android/viber.py | 30 +++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 64b9333432..0fd9a43e22 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -70,12 +70,12 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): #Extract TSK_CONTACT and TSK_CALLLOG information for contact_and_calllog_db in contact_and_calllog_dbs: - parser_helper = AppDBParserHelper(self._PARSER_NAME, + helper = AppDBParserHelper(self._PARSER_NAME, contact_and_calllog_db.getDBFile(), Account.Type.VIBER) contacts_parser = ViberContactsParser(contact_and_calllog_db) while contacts_parser.next(): - parser_helper.addContact( + helper.addContact( contacts_parser.get_account_name(), contacts_parser.get_contact_name(), contacts_parser.get_phone(), @@ -87,7 +87,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): calllog_parser = ViberCallLogsParser(contact_and_calllog_db) while calllog_parser.next(): - parser_helper.addCalllog( + helper.addCalllog( calllog_parser.get_account_name(), calllog_parser.get_call_direction(), calllog_parser.get_phone_number_from(), @@ -102,12 +102,12 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): #Extract TSK_MESSAGE information for message_db in message_dbs: - parser_helper = AppDBParserHelper(self._PARSER_NAME, + helper = AppDBParserHelper(self._PARSER_NAME, message_db.getDBFile(), Account.Type.VIBER) messages_parser = ViberMessagesParser(message_db) while messages_parser.next(): - parser_helper.addMessage( + helper.addMessage( messages_parser.get_message_type(), messages_parser.get_message_direction(), messages_parser.get_phone_number_from(), @@ -216,6 +216,26 @@ class ViberMessagesParser(TskMessagesParser): """ def __init__(self, message_db): + """ + For our purposes, the Viber datamodel is as follows: + - People can take part in N conversation(s). A conversation can have N + members and messages are exchanged in a conversation. + - Viber has a conversation table, a participant table (the people/members in the above + analogy) and a messages table. + - Each row of the participants table maps a person to a conversation_id + - Each row in the messages table has a from participant id and a conversation id. + + The query below does the following: + - The first two inner joins on participants and participants_info build + the 1 to many (N) mappings between the sender and the recipients for each + conversation_id. If a and b do private messaging, then 2 rows in the result + will be a -> b and b -> a. + If a, b, c, d are in a group, then 4 rows containing a -> b,c,d. b -> a,c,d. etc. + Participants_info is needed to get phone numbers. + - The result of the above step is a look up table for each message. Joining this result + onto the messages table lets us know which participant a message originated from and + everyone else that received it. + """ super(ViberMessagesParser, self).__init__(message_db.runQuery( """ SELECT convo_participants.from_number AS from_number, From db467414ba9d6dcf5474a2dec116eb6e616e979c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 11:12:25 -0400 Subject: [PATCH 08/55] updated comments --- InternalPythonModules/android/viber.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 0fd9a43e22..9f02cb7bfb 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -218,7 +218,7 @@ class ViberMessagesParser(TskMessagesParser): def __init__(self, message_db): """ For our purposes, the Viber datamodel is as follows: - - People can take part in N conversation(s). A conversation can have N + - People can take part in N conversation(s). A conversation can have M members and messages are exchanged in a conversation. - Viber has a conversation table, a participant table (the people/members in the above analogy) and a messages table. @@ -227,7 +227,7 @@ class ViberMessagesParser(TskMessagesParser): The query below does the following: - The first two inner joins on participants and participants_info build - the 1 to many (N) mappings between the sender and the recipients for each + the 1 to many (M) mappings between the sender and the recipients for each conversation_id. If a and b do private messaging, then 2 rows in the result will be a -> b and b -> a. If a, b, c, d are in a group, then 4 rows containing a -> b,c,d. b -> a,c,d. etc. From 19526ba5a4d037c1ff25945c717f9e095a14c880 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 12:09:00 -0400 Subject: [PATCH 09/55] Copied infrastructure into new branch --- .../android/ResultSetIterator.py | 35 ++++++++++ .../android/TskCallLogsParser.py | 58 ++++++++++++++++ .../android/TskContactsParser.py | 49 +++++++++++++ .../android/TskMessagesParser.py | 68 +++++++++++++++++++ 4 files changed, 210 insertions(+) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..66ea27eee3 --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,58 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self.INCOMING_CALL = "Incoming" + self.OUTGOING_CALL = "Outgoing" + self._DEFAULT_STRING = "" + + def get_account_name(self): + return self._DEFAULT_STRING + + def get_call_direction(self): + return self._DEFAULT_STRING + + def get_phone_number_from(self): + return self._DEFAULT_STRING + + def get_phone_number_to(self): + return self._DEFAULT_STRING + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_contact_name(self): + return self._DEFAULT_STRING diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..e3edbb25c8 --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,68 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self.INCOMING_MSG = "Incoming" + self.OUTGOING_MSG = "Outgoing" + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_TEXT + + def get_phone_number_from(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_MSG_READ_STATUS + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT From 7026b84b7aacfe973b81fffe5b1b9bb186a550eb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 12:13:31 -0400 Subject: [PATCH 10/55] Moved old whatsapp work into this branch and modified module.py to run --- InternalPythonModules/android/module.py | 3 +- InternalPythonModules/android/whatsapp.py | 203 ++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 InternalPythonModules/android/whatsapp.py diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..322700481c 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import whatsapp class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), whatsapp.WhatsAppAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py new file mode 100644 index 0000000000..cceff4494f --- /dev/null +++ b/InternalPythonModules/android/whatsapp.py @@ -0,0 +1,203 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskMessagesParser import TskMessagesParser +from TskContactsParser import TskContactsParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the WhatsApp databases for TSK contact and message artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._WHATSAPP_PACKAGE_NAME = "com.whatsapp" + self._PARSER_NAME = "WhatsApp Parser" + + def analyze(self, dataSource, fileManager, context): + """ + Extract, Transform and Load all messages and contacts from the WhatsApp databases. + """ + + try: + contact_dbs = SQLiteUtil.findAppDatabases(dataSource, "wa.db", self._WHATSAPP_PACKAGE_NAME) + message_dbs = SQLiteUtil.findAppDatabases(dataSource, "msgstore.db", self._WHATSAPP_PACKAGE_NAME) + + #Extract TSK_CONTACT information + for contact_db in contact_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_db.getDBFile(), Account.Type.WHATSAPP) + contacts_parser = WhatsAppContactsParser(contact_db) + while contacts_parser.next(): + blackboard_util.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + + for message_db in message_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, message_db.getDBFile(), Account.Type.WHATSAPP) + """ + message_db.attachDatabase(message_db.getDBFile().getParentPath(), "wa.db", "wadb") + messages_parser = WhatsAppMessagesParser(message_db) + while messages_parser.next(): + blackboard_util.addMessage( + messages_parser.get_account_id(), + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + """ + except (SQLException, TskCoreException) as ex: + #Error parsing WhatsApp db + self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + +class WhatsAppContactsParser(TskContactsParser): + """ + Extracts TSK_CONTACT information from the WhatsApp database. + TSK_CONTACT fields that are not in the WhatsApp database are given a default value + inherited from the super class. + """ + + def __init__(self, contact_db): + super(WhatsAppContactsParser, self).__init__(contact_db.runQuery( + """ + SELECT number, + CASE + WHEN given_name is NULL THEN family_name + WHEN family_name is NULL THEN given_name + ELSE given_name + || " " + || family_name + END name + FROM wa_contacts + WHERE given_name is not NULL OR family_name IS NOT NULL + """ + ) + ) + + def get_account_name(self): + return self.result_set.getString("name") + + def get_contact_name(self): + return self.result_set.getString("name") + + def get_phone(self): + return self.result_set.getString("number") + +class WhatsAppMessagesParser(TskMessagesParser): + """ + Extract TSK_MESSAGE information from the WhatsApp database. + TSK_CONTACT fields that are not in the WhatsApp database are given a default value + inherited from the super class. + """ + + def __init__(self, message_db): + super(WhatsAppMessageParser, self).__init__(message_db.runQuery( + """ + SELECT M.data AS content, + WDB.number AS number, + CASE + WHEN WDB.given_name IS NULL THEN WDB.family_name + WHEN WDB.family_name IS NULL THEN WDB.given_name + ELSE WDB.given_name + || " " + || WDB.family_name + END name, + M.key_from_me AS direction, + M.received_timestamp AS recieved_datetime, + M.timestamp AS send_datetime + FROM messages AS M + JOIN wadb.wa_contacts AS WDB + ON M.key_remote_jid = WDB.jid + """ + ) + ) + self._WHATSAPP_MESSAGE_TYPE = "WhatsApp Message" + self._INCOMING_MESSAGE_TYPE = 0 + self._OUTGOING_MESSAGE_TYPE = 1 + self._INCOMING_MSG_STRING = "Incoming" + self._OUTGOING_MSG_STRING = "Outgoing" + + def get_account_id(self): + return self.result_set.getString("name") + + def get_message_type(self): + return self._WHATSAPP_MESSAGE_TYPE + + def get_phone_number_to(self): + if self.get_message_direction() == self._OUTGOING_MSG_STRING: + return self.result_set.getString("number") + return super(WhatsAppMessageParser, self).get_phone_number_to() + + def get_phone_number_from(self): + if self.get_message_direction() == self._INCOMING_MSG_STRING: + return self.result_set.getString("number") + return super(WhatsAppMessageParser, self).get_phone_number_from() + + def get_message_direction(self): + direction = self.result_set.getInt("direction") + if direction == self._INCOMING_MESSAGE_TYPE: + return self._INCOMING_MSG_STRING + return self._OUTGOING_MSG_STRING + + def get_message_date_time(self): + #transform from ms to seconds + if get_message_direction() == self._OUTGOING_MSG_STRING: + return self.result_set.getLong("send_datetime") / 1000 + return self.result_set.getLong("received_datetime") / 1000 + + def get_message_text(self): + return self.result_set.getString("content") From a131ce17602dba62958aa8186023ada08514e12d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 15:08:43 -0400 Subject: [PATCH 11/55] Rest of the whats app implementation and some refactoring --- InternalPythonModules/android/whatsapp.py | 126 ++++++++++++---------- 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index cceff4494f..cc46bc0380 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -29,8 +29,8 @@ from java.util.logging import Level from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger -from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact @@ -57,19 +57,24 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): """ - Extract, Transform and Load all messages and contacts from the WhatsApp databases. + Extract, Transform and Load all TSK contact and message + artifacts from the WhatsApp databases. """ try: - contact_dbs = SQLiteUtil.findAppDatabases(dataSource, "wa.db", self._WHATSAPP_PACKAGE_NAME) - message_dbs = SQLiteUtil.findAppDatabases(dataSource, "msgstore.db", self._WHATSAPP_PACKAGE_NAME) + contact_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "wa.db", True, self._WHATSAPP_PACKAGE_NAME) + message_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "msgstore.db", True, self._WHATSAPP_PACKAGE_NAME) #Extract TSK_CONTACT information for contact_db in contact_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_db.getDBFile(), Account.Type.WHATSAPP) + helper = AppDBParserHelper(self._PARSER_NAME, + contact_db.getDBFile(), Account.Type.WHATSAPP) + contacts_parser = WhatsAppContactsParser(contact_db) while contacts_parser.next(): - blackboard_util.addContact( + helper.addContact( contacts_parser.get_account_name(), contacts_parser.get_contact_name(), contacts_parser.get_phone(), @@ -79,14 +84,18 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): ) contacts_parser.close() + contact_db.close() + for message_db in message_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, message_db.getDBFile(), Account.Type.WHATSAPP) - """ - message_db.attachDatabase(message_db.getDBFile().getParentPath(), "wa.db", "wadb") + helper = AppDBParserHelper(self._PARSER_NAME, + message_db.getDBFile(), Account.Type.WHATSAPP) + + message_db.attachDatabase(dataSource, "wa.db", + message_db.getDBFile().getParentPath(), "wadb") + messages_parser = WhatsAppMessagesParser(message_db) while messages_parser.next(): - blackboard_util.addMessage( - messages_parser.get_account_id(), + helper.addMessage( messages_parser.get_message_type(), messages_parser.get_message_direction(), messages_parser.get_phone_number_from(), @@ -98,7 +107,8 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - """ + + message_db.close() except (SQLException, TskCoreException) as ex: #Error parsing WhatsApp db self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) @@ -107,29 +117,21 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): class WhatsAppContactsParser(TskContactsParser): """ Extracts TSK_CONTACT information from the WhatsApp database. - TSK_CONTACT fields that are not in the WhatsApp database are given a default value - inherited from the super class. + TSK_CONTACT fields that are not in the WhatsApp database are given + a default value inherited from the super class. """ def __init__(self, contact_db): super(WhatsAppContactsParser, self).__init__(contact_db.runQuery( """ - SELECT number, - CASE - WHEN given_name is NULL THEN family_name - WHEN family_name is NULL THEN given_name - ELSE given_name - || " " - || family_name - END name - FROM wa_contacts - WHERE given_name is not NULL OR family_name IS NOT NULL + SELECT """ + _get_contacts_formatting() + """ + FROM wa_contacts AS WC """ ) ) def get_account_name(self): - return self.result_set.getString("name") + return self.get_phone() def get_contact_name(self): return self.result_set.getString("name") @@ -140,64 +142,80 @@ class WhatsAppContactsParser(TskContactsParser): class WhatsAppMessagesParser(TskMessagesParser): """ Extract TSK_MESSAGE information from the WhatsApp database. - TSK_CONTACT fields that are not in the WhatsApp database are given a default value - inherited from the super class. + TSK_CONTACT fields that are not in the WhatsApp database are given + a default value inherited from the super class. """ def __init__(self, message_db): - super(WhatsAppMessageParser, self).__init__(message_db.runQuery( + super(WhatsAppMessagesParser, self).__init__(message_db.runQuery( """ SELECT M.data AS content, - WDB.number AS number, - CASE - WHEN WDB.given_name IS NULL THEN WDB.family_name - WHEN WDB.family_name IS NULL THEN WDB.given_name - ELSE WDB.given_name - || " " - || WDB.family_name - END name, + """+_get_contacts_formatting()+""", M.key_from_me AS direction, - M.received_timestamp AS recieved_datetime, + M.received_timestamp AS received_datetime, M.timestamp AS send_datetime FROM messages AS M - JOIN wadb.wa_contacts AS WDB - ON M.key_remote_jid = WDB.jid + JOIN wadb.wa_contacts AS WC + ON M.key_remote_jid = WC.jid """ ) ) self._WHATSAPP_MESSAGE_TYPE = "WhatsApp Message" self._INCOMING_MESSAGE_TYPE = 0 self._OUTGOING_MESSAGE_TYPE = 1 - self._INCOMING_MSG_STRING = "Incoming" - self._OUTGOING_MSG_STRING = "Outgoing" - - def get_account_id(self): - return self.result_set.getString("name") def get_message_type(self): return self._WHATSAPP_MESSAGE_TYPE def get_phone_number_to(self): - if self.get_message_direction() == self._OUTGOING_MSG_STRING: - return self.result_set.getString("number") - return super(WhatsAppMessageParser, self).get_phone_number_to() + if self.get_message_direction() == self.OUTGOING_MSG: + return Account.Address(self.result_set.getString("number"), + self.result_set.getString("number")) + return super(WhatsAppMessagesParser, self).get_phone_number_to() def get_phone_number_from(self): - if self.get_message_direction() == self._INCOMING_MSG_STRING: - return self.result_set.getString("number") - return super(WhatsAppMessageParser, self).get_phone_number_from() + if self.get_message_direction() == self.INCOMING_MSG: + return Account.Address(self.result_set.getString("number"), + self.result_set.getString("number")) + return super(WhatsAppMessagesParser, self).get_phone_number_from() def get_message_direction(self): direction = self.result_set.getInt("direction") if direction == self._INCOMING_MESSAGE_TYPE: - return self._INCOMING_MSG_STRING - return self._OUTGOING_MSG_STRING + return self.INCOMING_MSG + return self.OUTGOING_MSG def get_message_date_time(self): #transform from ms to seconds - if get_message_direction() == self._OUTGOING_MSG_STRING: + if self.get_message_direction() == self.OUTGOING_MSG: return self.result_set.getLong("send_datetime") / 1000 return self.result_set.getLong("received_datetime") / 1000 def get_message_text(self): return self.result_set.getString("content") + +def _get_contacts_formatting(): + """ + This function is here to explicitly stress the point that the + formatting routine used in the contacts and messages parsers + should never differ. These fields are used to correlate in Autopsy. + + The SQL statement assumes wa_contacts table is named WC. + """ + + return """ + CASE + WHEN WC.number IS NULL THEN WC.jid + WHEN WC.number == "" THEN WC.jid + ELSE WC.number + END number, + CASE + WHEN WC.given_name IS NULL + AND WC.family_name IS NULL THEN WC.jid + WHEN WC.given_name IS NULL THEN WC.family_name + WHEN WC.family_name IS NULL THEN WC.given_name + ELSE WC.given_name + || " " + || WC.family_name + END name + """ From 0580ffdd2fb47b23d5adc6c0a060fc9a15e661ab Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 16:39:48 -0400 Subject: [PATCH 12/55] Merged lasted helper code and updated viber.py to conform --- .../android/TskCallLogsParser.py | 25 ++++++++----- .../android/TskMessagesParser.py | 10 +++-- InternalPythonModules/android/viber.py | 37 ++++++++++++------- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 66ea27eee3..3bc0e0141b 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): """ @@ -32,27 +34,30 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) - self.INCOMING_CALL = "Incoming" - self.OUTGOING_CALL = "Outgoing" self._DEFAULT_STRING = "" + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = Account.Address("","") + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - def get_account_name(self): - return self._DEFAULT_STRING + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO def get_call_direction(self): - return self._DEFAULT_STRING + return self._DEFAULT_DIRECTION def get_phone_number_from(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_call_start_date_time(self): return self._DEFAULT_LONG def get_call_end_date_time(self): return self._DEFAULT_LONG - - def get_contact_name(self): - return self._DEFAULT_STRING + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index e3edbb25c8..69d05cd6fe 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -33,18 +33,22 @@ class TskMessagesParser(ResultSetIterator): def __init__(self, result_set): super(TskMessagesParser, self).__init__(result_set) - self.INCOMING_MSG = "Incoming" - self.OUTGOING_MSG = "Outgoing" self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD def get_message_type(self): return self._DEFAULT_TEXT def get_message_direction(self): - return self._DEFAULT_TEXT + return self._DEFAULT_COMMUNICATION_DIRECTION def get_phone_number_from(self): return self._DEFAULT_ACCOUNT_ADDRESS diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 9f02cb7bfb..a8442a2931 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -88,13 +88,12 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): calllog_parser = ViberCallLogsParser(contact_and_calllog_db) while calllog_parser.next(): helper.addCalllog( - calllog_parser.get_account_name(), calllog_parser.get_call_direction(), calllog_parser.get_phone_number_from(), calllog_parser.get_phone_number_to(), calllog_parser.get_call_start_date_time(), calllog_parser.get_call_end_date_time(), - calllog_parser.get_contact_name() + calllog_parser.get_call_type() ) calllog_parser.close() @@ -139,7 +138,8 @@ class ViberCallLogsParser(TskCallLogsParser): SELECT C.canonized_number AS number, C.type AS direction, C.duration AS seconds, - C.date AS start_time + C.date AS start_time, + C.viber_call_type AS call_type FROM calls AS C """ ) @@ -148,20 +148,21 @@ class ViberCallLogsParser(TskCallLogsParser): self._OUTGOING_CALL_TYPE = 2 self._INCOMING_CALL_TYPE = 1 self._MISSED_CALL_TYPE = 3 - - def get_account_name(self): - return self.result_set.getString("number") + self._AUDIO_CALL_TYPE = 1 + self._VIDEO_CALL_TYPE = 4 def get_phone_number_from(self): if self.get_call_direction() == self.INCOMING_CALL: - return self.result_set.getString("number") + return Account.Address(self.result_set.getString("number"), + self.result_set.getString("number")) #Give default value if the call is outgoing, #the device's # is not stored in the database. return super(ViberCallLogsParser, self).get_phone_number_from() def get_phone_number_to(self): if self.get_call_direction() == self.OUTGOING_CALL: - return self.result_set.getString("number") + return Account.Address(self.result_set.getString("number"), + self.result_set.getString("number")) #Give default value if the call is incoming, #the device's # is not stored in the database. return super(ViberCallLogsParser, self).get_phone_number_to() @@ -169,7 +170,7 @@ class ViberCallLogsParser(TskCallLogsParser): def get_call_direction(self): direction = self.result_set.getInt("direction") if direction == self._INCOMING_CALL_TYPE or direction == self._MISSED_CALL_TYPE: - return self.INCOMING_CALL + return self.INCOMING_CALL return self.OUTGOING_CALL def get_call_start_date_time(self): @@ -180,6 +181,14 @@ class ViberCallLogsParser(TskCallLogsParser): duration = self.result_set.getLong("seconds") return start_time + duration + def get_call_type(self): + call_type = self.result_set.getInt("call_type") + if call_type == self._AUDIO_CALL_TYPE: + return self.AUDIO_CALL + if call_type == self._VIDEO_CALL_TYPE: + return self.VIDEO_CALL + return super(ViberCallLogsParser, self).get_call_type() + class ViberContactsParser(TskContactsParser): """ Extracts TSK_CONTACT information from the Viber database. @@ -282,8 +291,8 @@ class ViberMessagesParser(TskMessagesParser): def get_message_direction(self): direction = self.result_set.getInt("direction") if direction == self._INCOMING_MESSAGE_TYPE: - return self.INCOMING_MSG - return self.OUTGOING_MSG + return self.INCOMING + return self.OUTGOING def get_phone_number_to(self): recipients = [] @@ -296,11 +305,11 @@ class ViberMessagesParser(TskMessagesParser): return self.result_set.getLong("msg_date") / 1000 def get_message_read_status(self): - if self.get_message_direction() == self.INCOMING_MSG: + if self.get_message_direction() == self.INCOMING: if self.result_set.getInt("read_status") == 0: - return AppDBParserHelper.MessageReadStatusEnum.READ + return self.READ else: - return AppDBParserHelper.MessageReadStatusEnum.UNREAD + return self.UNREAD return super(ViberMessagesParser, self).get_message_read_status() def get_message_text(self): From 0213e40d3ef2321354a47c8982e3661704388844 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 16:48:21 -0400 Subject: [PATCH 13/55] Initial infra commit --- .../android/ResultSetIterator.py | 35 +++++++++ .../android/TskCallLogsParser.py | 63 ++++++++++++++++ .../android/TskContactsParser.py | 49 +++++++++++++ .../android/TskMessagesParser.py | 72 +++++++++++++++++++ 4 files changed, 219 insertions(+) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..3bc0e0141b --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,63 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self._DEFAULT_STRING = "" + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = Account.Address("","") + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + + def get_call_direction(self): + return self._DEFAULT_DIRECTION + + def get_phone_number_from(self): + return self._DEFAULT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ADDRESS + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..69d05cd6fe --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,72 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_COMMUNICATION_DIRECTION + + def get_phone_number_from(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_MSG_READ_STATUS + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT From 7c15ebc260529d5307b027a9deaed329abd56343 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 16:58:19 -0400 Subject: [PATCH 14/55] Initial template commit --- InternalPythonModules/android/textnow.py | 195 +++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 InternalPythonModules/android/textnow.py diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py new file mode 100644 index 0000000000..a84d1a3931 --- /dev/null +++ b/InternalPythonModules/android/textnow.py @@ -0,0 +1,195 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskMessagesParser import TskMessagesParser +from TskContactsParser import TskContactsParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class TextNowAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the TextNow App databases for TSK contacts, message + and calllog artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._TEXTNOW_PACKAGE_NAME = "com.enflick.android.TextNow" + self._PARSER_NAME = "TextNow Parser" + + def analyze(self, dataSource, fileManager, context): + """ + Extract, Transform and Load all messages, contacts and + calllogs from the TextNow databases. + """ + + try: + textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "viber_data", True, self._TEXTNOW_PACKAGE_NAME) + + #Extract TSK_CONTACT and TSK_CALLLOG information + for textnow_db in textnow_dbs: + helper = AppDBParserHelper(self._PARSER_NAME, + textnow_db.getDBFile(), Account.Type.TEXTNOW) + + contacts_parser = TextNowContactsParser(textnow_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + + calllog_parser = TextNowCallLogsParser(textnow_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + + messages_parser = TextNowMessagesParser(textnow_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + + textnow_db.close() + except (SQLException, TskCoreException) as ex: + #Error parsing TextNow db + self._logger.log(Level.WARNING, "Error parsing TextNow Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + +class TextNowCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from the TextNow database. + TSK_CALLLOG fields that are not in the TextNow database are given + a default value inherited from the super class. + """ + + def __init__(self, calllog_db): + super(TextNowCallLogsParser, self).__init__(calllog_db.runQuery( + """ + """ + ) + ) + + def get_phone_number_from(self): + + def get_phone_number_to(self): + + def get_call_direction(self): + + def get_call_start_date_time(self): + + def get_call_end_date_time(self): + + def get_call_type(self): + +class TextNowContactsParser(TskContactsParser): + """ + Extracts TSK_CONTACT information from the TextNow database. + TSK_CONTACT fields that are not in the TextNow database are given + a default value inherited from the super class. + """ + + def __init__(self, contact_db): + super(TextNowContactsParser, self).__init__(contact_db.runQuery( + """ + """ + ) + ) + + def get_account_name(self): + + def get_contact_name(self): + + def get_phone(self): + +class TextNowMessagesParser(TskMessagesParser): + """ + Extract TSK_MESSAGE information from the TextNow database. + TSK_CONTACT fields that are not in the TextNow database are given + a default value inherited from the super class. + """ + + def __init__(self, message_db): + super(TextNowMessagesParser, self).__init__(message_db.runQuery( + """ + """ + ) + ) + self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" + + def get_message_type(self): + return self._TEXTNOW_MESSAGE_TYPE + + def get_phone_number_from(self): + + def get_message_direction(self): + + def get_phone_number_to(self): + + def get_message_date_time(self): + + def get_message_read_status(self): + + def get_message_text(self): + + def get_thread_id(self): From 4f171c7a51361c75918fa0f0b9a0b0ae53612c84 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Sep 2019 17:19:14 -0400 Subject: [PATCH 15/55] Updated parser templates with correct default values --- InternalPythonModules/android/TskCallLogsParser.py | 2 +- InternalPythonModules/android/TskMessagesParser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 3bc0e0141b..763ba3c15f 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -36,7 +36,7 @@ class TskCallLogsParser(ResultSetIterator): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN - self._DEFAULT_ADDRESS = Account.Address("","") + self._DEFAULT_ADDRESS = None self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 69d05cd6fe..15c4166db7 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -36,7 +36,7 @@ class TskMessagesParser(ResultSetIterator): self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN - self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_ACCOUNT_ADDRESS = None self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING From e90a2ea15c0e81538fe188e9d8a1dcc952cf01ea Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Sep 2019 17:46:53 -0400 Subject: [PATCH 16/55] More infra additions --- InternalPythonModules/android/general.py | 11 +++++++++++ InternalPythonModules/android/module.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..f3dce7b96a 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import textnow class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), textnow.TextNowAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) From 8f3eab11a421bf01c0483424520d0ae76ea6047d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Sep 2019 17:47:14 -0400 Subject: [PATCH 17/55] Fully implemented text now parser --- InternalPythonModules/android/textnow.py | 136 ++++++++++++++++++++++- 1 file changed, 131 insertions(+), 5 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index a84d1a3931..6ab7d14801 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -41,6 +41,7 @@ from org.sleuthkit.datamodel import Account from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser +from general import appendAttachmentList import traceback import general @@ -55,6 +56,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) self._TEXTNOW_PACKAGE_NAME = "com.enflick.android.TextNow" self._PARSER_NAME = "TextNow Parser" + self._VERSION = "6.41.0.2" def analyze(self, dataSource, fileManager, context): """ @@ -64,13 +66,13 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): try: textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "viber_data", True, self._TEXTNOW_PACKAGE_NAME) + "textnow_data.db", True, self._TEXTNOW_PACKAGE_NAME) - #Extract TSK_CONTACT and TSK_CALLLOG information for textnow_db in textnow_dbs: helper = AppDBParserHelper(self._PARSER_NAME, textnow_db.getDBFile(), Account.Type.TEXTNOW) + #Extract TSK_CONTACT information contacts_parser = TextNowContactsParser(textnow_db) while contacts_parser.next(): helper.addContact( @@ -83,6 +85,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): ) contacts_parser.close() + #Extract TSK_CALLLOG information calllog_parser = TextNowCallLogsParser(textnow_db) while calllog_parser.next(): helper.addCalllog( @@ -95,6 +98,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): ) calllog_parser.close() + #Extract TSK_MESSAGES information messages_parser = TextNowMessagesParser(textnow_db) while messages_parser.next(): helper.addMessage( @@ -109,7 +113,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - + textnow_db.close() except (SQLException, TskCoreException) as ex: #Error parsing TextNow db @@ -126,21 +130,47 @@ class TextNowCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(TextNowCallLogsParser, self).__init__(calllog_db.runQuery( """ + SELECT contact_value AS num, + message_direction AS direction, + message_text AS duration, + date AS datetime + FROM messages AS M + WHERE message_type IN ( 100, 102 ) """ ) ) + self._INCOMING_CALL_TYPE = 1 + self._OUTGOING_CALL_TYPE = 2 + self._has_errors = False def get_phone_number_from(self): + if self.get_call_direction() == self.OUTGOING_CALL: + return super(TextNowCallLogsParser, self).get_phone_number_from() + return Account.Address(self.result_set.getString("num"), + self.result_set.getString("num")) def get_phone_number_to(self): + if self.get_call_direction() == self.INCOMING_CALL: + return super(TextNowCallLogsParser, self).get_phone_number_to() + return Account.Address(self.result_set.getString("num"), + self.result_set.getString("num")) def get_call_direction(self): + if self.result_set.getInt("direction") == self._INCOMING_CALL_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL def get_call_start_date_time(self): + return self.result_set.getLong("datetime") / 1000 def get_call_end_date_time(self): - - def get_call_type(self): + start = self.get_call_start_date_time() + duration = self.result_set.getString("duration") + try: + return start + long(duration) + except ValueError as ve: + self._has_errors = True + return super(TextNowCallLogsParser, self).get_call_end_date_time() class TextNowContactsParser(TskContactsParser): """ @@ -152,15 +182,25 @@ class TextNowContactsParser(TskContactsParser): def __init__(self, contact_db): super(TextNowContactsParser, self).__init__(contact_db.runQuery( """ + SELECT C.contact_value AS number, + CASE + WHEN contact_name IS NULL THEN contact_value + WHEN contact_name == "" THEN contact_value + ELSE contact_name + END name + FROM contacts AS C """ ) ) def get_account_name(self): + return self.result_set.getString("number") def get_contact_name(self): + return self.result_set.getString("name") def get_phone(self): + return self.result_set.getString("number") class TextNowMessagesParser(TskMessagesParser): """ @@ -170,26 +210,112 @@ class TextNowMessagesParser(TskMessagesParser): """ def __init__(self, message_db): + """ + The TextNow database in v6.41.0.2 is structured as follows: + - A messages table, which stores messages from/to a number + - A contacts table, which stores phone numbers + - A groups table, which stores each group the device owner is a part of + - A group_members table, which stores who is in each group + + The messages table contains both call logs and messages, with a type + column differentiating the two. + + The query below does the following: + - The group_info inner query creates a comma seperated list of group recipients + for each group. This result is then joined on the groups table to get the thread id. + - The contacts table is unioned with this result so we have a complete map + of "from" phone_numbers -> recipients (group or single). This is the + 'to_from_map' inner query. + - Finally, the to_from_map results are joined with the messages table to get all + of the communication details. + """ super(TextNowMessagesParser, self).__init__(message_db.runQuery( """ + SELECT CASE + WHEN message_direction == 2 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE contact_name + end from_address, + CASE + WHEN message_direction == 1 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE to_addresses + end to_address, + message_direction, + message_text, + M.READ, + M.date, + M.attach, + thread_id + FROM (SELECT group_info.contact_value, + group_info.to_addresses, + G._id AS thread_id + FROM (SELECT GM.contact_value, + Group_concat(GM.member_contact_value) AS to_addresses + FROM group_members AS GM + GROUP BY GM.contact_value) AS group_info + JOIN groups AS G + ON G.contact_value = group_info.contact_value + UNION + SELECT c.contact_value, + NULL, + -1 + FROM contacts AS c) AS to_from_map + JOIN messages AS M + ON M.contact_value = to_from_map.contact_value + WHERE message_type NOT IN ( 102, 100 ) """ ) ) self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" + self._INCOMING_MESSAGE_TYPE = 1 + self._OUTGOING_MESSAGE_TYPE = 2 + self._UNKNOWN_THREAD_ID = -1 def get_message_type(self): return self._TEXTNOW_MESSAGE_TYPE def get_phone_number_from(self): + if self.result_set.getString("from_address") == "": + return super(TextNowMessagesParser, self).get_phone_number_from() + return Account.Address(self.result_set.getString("from_address"), + self.result_set.getString("from_address")) def get_message_direction(self): + direction = self.result_set.getInt("message_direction") + if direction == self._INCOMING_MESSAGE_TYPE: + return self.INCOMING + return self.OUTGOING def get_phone_number_to(self): + if self.result_set.getString("to_address") == "": + return super(TextNowMessagesParser, self).get_phone_number_to() + return Account.Address(self.result_set.getString("to_address"), + self.result_set.getString("to_address")) def get_message_date_time(self): + #convert ms to s + return self.result_set.getLong("date") / 1000; def get_message_read_status(self): + read = self.result_set.getBoolean("read") + if self.get_message_direction() == self.INCOMING: + if read == True: + return self.READ + return self.UNREAD + + #read status for outgoing messages cannot be determined, give default + return super(TextNowMessagesParser, self).get_message_read_status() def get_message_text(self): + text = self.result_set.getString("message_text") + attachment = self.result_set.getString("attach") + if attachment != "": + text = appendAttachmentList(text, [attachment]) + return text def get_thread_id(self): + thread_id = self.result_set.getInt("thread_id") + if thread_id == self._UNKNOWN_THREAD_ID: + return super(TextNowMessagesParser, self).get_thread_id() + return str(thread_id) From 6d1214261a0ad8354d8ecbbfdd9a3b0173965bb1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Sep 2019 17:53:31 -0400 Subject: [PATCH 18/55] infra updates --- InternalPythonModules/android/TskCallLogsParser.py | 2 +- InternalPythonModules/android/TskMessagesParser.py | 2 +- InternalPythonModules/android/general.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 3bc0e0141b..763ba3c15f 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -36,7 +36,7 @@ class TskCallLogsParser(ResultSetIterator): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN - self._DEFAULT_ADDRESS = Account.Address("","") + self._DEFAULT_ADDRESS = None self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 69d05cd6fe..15c4166db7 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -36,7 +36,7 @@ class TskMessagesParser(ResultSetIterator): self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN - self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_ACCOUNT_ADDRESS = None self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body From fa9210114a6087a7c65b3965893a32724a877ad3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sat, 14 Sep 2019 10:29:06 -0400 Subject: [PATCH 19/55] Added version number --- InternalPythonModules/android/viber.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index a8442a2931..e6949b169e 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -55,6 +55,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) self._VIBER_PACKAGE_NAME = "com.viber.voip" self._PARSER_NAME = "Viber Parser" + self._VERSION = "11.5.0" def analyze(self, dataSource, fileManager, context): """ From 8fd7abdf559e9e965b1f2e827b25e618e627da6b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sat, 14 Sep 2019 11:03:49 -0400 Subject: [PATCH 20/55] Updated infra changes --- .../android/TskCallLogsParser.py | 25 +++++++++++-------- .../android/TskMessagesParser.py | 12 ++++++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 66ea27eee3..763ba3c15f 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): """ @@ -32,27 +34,30 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) - self.INCOMING_CALL = "Incoming" - self.OUTGOING_CALL = "Outgoing" self._DEFAULT_STRING = "" + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = None + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - def get_account_name(self): - return self._DEFAULT_STRING + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO def get_call_direction(self): - return self._DEFAULT_STRING + return self._DEFAULT_DIRECTION def get_phone_number_from(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_call_start_date_time(self): return self._DEFAULT_LONG def get_call_end_date_time(self): return self._DEFAULT_LONG - - def get_contact_name(self): - return self._DEFAULT_STRING + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index e3edbb25c8..15c4166db7 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -33,18 +33,22 @@ class TskMessagesParser(ResultSetIterator): def __init__(self, result_set): super(TskMessagesParser, self).__init__(result_set) - self.INCOMING_MSG = "Incoming" - self.OUTGOING_MSG = "Outgoing" self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN - self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_ACCOUNT_ADDRESS = None + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD def get_message_type(self): return self._DEFAULT_TEXT def get_message_direction(self): - return self._DEFAULT_TEXT + return self._DEFAULT_COMMUNICATION_DIRECTION def get_phone_number_from(self): return self._DEFAULT_ACCOUNT_ADDRESS From 611a03fa09e1e0af763e5fbe73f81eb6e923cb0b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sat, 14 Sep 2019 11:04:52 -0400 Subject: [PATCH 21/55] Added version number --- InternalPythonModules/android/whatsapp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index cc46bc0380..7afdf184c2 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -54,6 +54,7 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) self._WHATSAPP_PACKAGE_NAME = "com.whatsapp" self._PARSER_NAME = "WhatsApp Parser" + self._VERSION = "2.19.244" def analyze(self, dataSource, fileManager, context): """ From 7f2464a2e59d6c7dd83b9aa6e5147587650bb448 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 12:21:05 -0400 Subject: [PATCH 22/55] Fully implemented the whatsapp parser --- InternalPythonModules/android/general.py | 11 + InternalPythonModules/android/whatsapp.py | 279 ++++++++++++++++++---- 2 files changed, 238 insertions(+), 52 deletions(-) diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 7afdf184c2..9ae57c09d6 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -41,13 +41,15 @@ from org.sleuthkit.datamodel import Account from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser +from general import appendAttachmentList import traceback import general class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): """ - Parses the WhatsApp databases for TSK contact and message artifacts. + Parses the WhatsApp databases for TSK contact, message + and calllog artifacts. """ def __init__(self): @@ -58,8 +60,8 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): """ - Extract, Transform and Load all TSK contact and message - artifacts from the WhatsApp databases. + Extract, Transform and Load all TSK contact, message + and calllog artifacts from the WhatsApp databases. """ try: @@ -109,12 +111,154 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): ) messages_parser.close() + group_calllogs_parser = WhatsAppGroupCallLogsParser(message_db) + while group_calllogs_parser.next(): + helper.addCalllog( + group_calllogs_parser.get_call_direction(), + group_calllogs_parser.get_phone_number_from(), + group_calllogs_parser.get_phone_number_to(), + group_calllogs_parser.get_call_start_date_time(), + group_calllogs_parser.get_call_end_date_time(), + group_calllogs_parser.get_call_type() + ) + group_calllogs_parser.close() + + single_calllogs_parser = WhatsAppSingleCallLogsParser(message_db) + while single_calllogs_parser.next(): + helper.addCalllog( + single_calllogs_parser.get_call_direction(), + single_calllogs_parser.get_phone_number_from(), + single_calllogs_parser.get_phone_number_to(), + single_calllogs_parser.get_call_start_date_time(), + single_calllogs_parser.get_call_end_date_time(), + single_calllogs_parser.get_call_type() + ) + single_calllogs_parser.close() + message_db.close() except (SQLException, TskCoreException) as ex: #Error parsing WhatsApp db self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) self._logger.log(Level.WARNING, traceback.format_exec()) +class WhatsAppGroupCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from group call logs + in the WhatsApp database. + """ + + def __init__(self, calllog_db): + super(WhatsAppGroupCallLogsParser, self).__init__(calllog_db.runQuery( + """ + SELECT CL.video_call, + CL.timestamp, + CL.duration, + CL.from_me, + J.raw_string as from_num, + group_concat(J.raw_string) AS group_members + FROM call_log_participant_v2 AS CLP + JOIN call_log AS CL + ON CL._id = CLP.call_log_row_id + JOIN jid AS J + ON J._id = CLP.jid_row_id + GROUP BY CL._id + """ + ) + ) + self._INCOMING_CALL_TYPE = 0 + self._OUTGOING_CALL_TYPE = 1 + self._VIDEO_CALL_TYPE = 1 + + def get_call_direction(self): + if self.result_set.getInt("from_me") == self._INCOMING_CALL_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL + + def get_phone_number_from(self): + if self.get_call_direction() == self.INCOMING_CALL: + sender = self.result_set.getString("from_num") + return Account.Address(sender, sender) + return super(WhatsAppGroupCallLogsParser, self).get_phone_number_from() + + def get_phone_number_to(self): + if self.get_call_direction() == self.OUTGOING_CALL: + group = self.result_set.getString("group_members") + members = [] + for token in group.split(","): + members.append(Account.Address(token, token)) + return members + return super(WhatsAppGroupCallLogsParser, self).get_phone_number_to() + + def get_call_start_date_time(self): + return self.result_set.getLong("timestamp") / 1000 + + def get_call_end_date_time(self): + start = self.get_call_start_date_time() + duration = self.result_set.getInt("duration") + return start + duration + + def get_call_type(self): + if self.result_set.getInt("video_call") == self._VIDEO_CALL_TYPE: + return self.VIDEO_CALL + return self.AUDIO_CALL + +class WhatsAppSingleCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from 1 to 1 call logs + in the WhatsApp database. + """ + + def __init__(self, calllog_db): + super(WhatsAppSingleCallLogsParser, self).__init__(calllog_db.runQuery( + """ + SELECT CL.timestamp, + CL.video_call, + CL.duration, + J.raw_string AS num, + CL.from_me + FROM call_log AS CL + JOIN jid AS J + ON J._id = CL.jid_row_id + WHERE CL._id NOT IN (SELECT DISTINCT call_log_row_id + FROM call_log_participant_v2) + """ + ) + ) + self._INCOMING_CALL_TYPE = 0 + self._OUTGOING_CALL_TYPE = 1 + self._VIDEO_CALL_TYPE = 1 + + def get_call_direction(self): + if self.result_set.getInt("from_me") == self._INCOMING_CALL_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL + + def get_phone_number_from(self): + if self.get_call_direction() == self.INCOMING_CALL: + sender = self.result_set.getString("num") + return Account.Address(sender, sender) + return super(WhatsAppSingleCallLogsParser, self).get_phone_number_from() + + def get_phone_number_to(self): + if self.get_call_direction() == self.OUTGOING_CALL: + to = self.result_set.getString("num") + return Account.Address(to, to) + return super(WhatsAppSingleCallLogsParser, self).get_phone_number_to() + + def get_call_start_date_time(self): + return self.result_set.getLong("timestamp") / 1000 + + def get_call_end_date_time(self): + start = self.get_call_start_date_time() + duration = self.result_set.getInt("duration") + return start + duration + + def get_call_type(self): + if self.result_set.getInt("video_call") == self._VIDEO_CALL_TYPE: + return self.VIDEO_CALL + return self.AUDIO_CALL + + class WhatsAppContactsParser(TskContactsParser): """ Extracts TSK_CONTACT information from the WhatsApp database. @@ -125,14 +269,31 @@ class WhatsAppContactsParser(TskContactsParser): def __init__(self, contact_db): super(WhatsAppContactsParser, self).__init__(contact_db.runQuery( """ - SELECT """ + _get_contacts_formatting() + """ + SELECT jid, + CASE + WHEN WC.number IS NULL THEN WC.jid + WHEN WC.number == "" THEN WC.jid + ELSE WC.number + END number, + CASE + WHEN WC.given_name IS NULL + AND WC.family_name IS NULL + AND WC.display_name IS NULL THEN WC.jid + WHEN WC.given_name IS NULL + AND WC.family_name IS NULL THEN WC.display_name + WHEN WC.given_name IS NULL THEN WC.family_name + WHEN WC.family_name IS NULL THEN WC.given_name + ELSE WC.given_name + || " " + || WC.family_name + END name FROM wa_contacts AS WC """ - ) + ) ) def get_account_name(self): - return self.get_phone() + return self.result_set.getString("jid") def get_contact_name(self): return self.result_set.getString("name") @@ -150,73 +311,87 @@ class WhatsAppMessagesParser(TskMessagesParser): def __init__(self, message_db): super(WhatsAppMessagesParser, self).__init__(message_db.runQuery( """ - SELECT M.data AS content, - """+_get_contacts_formatting()+""", - M.key_from_me AS direction, - M.received_timestamp AS received_datetime, - M.timestamp AS send_datetime - FROM messages AS M - JOIN wadb.wa_contacts AS WC - ON M.key_remote_jid = WC.jid + SELECT M.key_remote_jid AS id, + contact_info.recipients, + key_from_me AS direction, + CASE + WHEN M.data IS NULL THEN "" + ELSE M.data + END AS content, + M.timestamp AS send_timestamp, + M.received_timestamp, + M.remote_resource AS group_sender, + M.media_url As attachment, + M.media_mime_type as attachment_mimetype + FROM (SELECT jid, + recipients + FROM wadb.wa_contacts AS WC + LEFT JOIN (SELECT gjid, + group_concat(CASE + WHEN jid == "" THEN NULL + ELSE jid + END) AS recipients + FROM group_participants + GROUP BY gjid) AS group_map + ON WC.jid = group_map.gjid + GROUP BY jid) AS contact_info + JOIN messages AS M + ON M.key_remote_jid = contact_info.jid """ ) ) self._WHATSAPP_MESSAGE_TYPE = "WhatsApp Message" self._INCOMING_MESSAGE_TYPE = 0 self._OUTGOING_MESSAGE_TYPE = 1 + self._message_db = message_db def get_message_type(self): return self._WHATSAPP_MESSAGE_TYPE def get_phone_number_to(self): - if self.get_message_direction() == self.OUTGOING_MSG: - return Account.Address(self.result_set.getString("number"), - self.result_set.getString("number")) + group = self.result_set.getString("recipients") + if group is not None: + return Account.Address(self.result_set.getString("id"), group) + if self.get_message_direction() == self.OUTGOING: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("id")) return super(WhatsAppMessagesParser, self).get_phone_number_to() def get_phone_number_from(self): - if self.get_message_direction() == self.INCOMING_MSG: - return Account.Address(self.result_set.getString("number"), - self.result_set.getString("number")) + if self.get_message_direction() == self.INCOMING: + group_sender = self.result_set.getString("group_sender") + group = self.result_set.getString("recipients") + if group_sender is not None and group is not None: + return Account.Address(group_sender, group_sender) + else: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("id")) return super(WhatsAppMessagesParser, self).get_phone_number_from() def get_message_direction(self): direction = self.result_set.getInt("direction") if direction == self._INCOMING_MESSAGE_TYPE: - return self.INCOMING_MSG - return self.OUTGOING_MSG + return self.INCOMING + return self.OUTGOING def get_message_date_time(self): #transform from ms to seconds - if self.get_message_direction() == self.OUTGOING_MSG: - return self.result_set.getLong("send_datetime") / 1000 - return self.result_set.getLong("received_datetime") / 1000 + if self.get_message_direction() == self.OUTGOING: + return self.result_set.getLong("send_timestamp") / 1000 + return self.result_set.getLong("received_timestamp") / 1000 def get_message_text(self): - return self.result_set.getString("content") - -def _get_contacts_formatting(): - """ - This function is here to explicitly stress the point that the - formatting routine used in the contacts and messages parsers - should never differ. These fields are used to correlate in Autopsy. - - The SQL statement assumes wa_contacts table is named WC. - """ - - return """ - CASE - WHEN WC.number IS NULL THEN WC.jid - WHEN WC.number == "" THEN WC.jid - ELSE WC.number - END number, - CASE - WHEN WC.given_name IS NULL - AND WC.family_name IS NULL THEN WC.jid - WHEN WC.given_name IS NULL THEN WC.family_name - WHEN WC.family_name IS NULL THEN WC.given_name - ELSE WC.given_name - || " " - || WC.family_name - END name - """ + message = self.result_set.getString("content") + attachment = self.result_set.getString("attachment") + if attachment is not None: + mime_type = self.result_set.getString("attachment_mimetype") + if mime_type is not None: + attachment += "\nMIME type: " + mime_type + return appendAttachmentList(message, [attachment]) + return message + + def get_thread_id(self): + group = self.result_set.getString("recipients") + if group is not None: + return self.result_set.getString("id") + return super(WhatsAppMessagesParser, self).get_thread_id() From 8edaab679e626270263368758357d5a212efcd44 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 12:32:35 -0400 Subject: [PATCH 23/55] infra upgrades --- .../android/TskCallLogsParser.py | 26 +++++++++++-------- .../android/TskMessagesParser.py | 22 +++++++++------- InternalPythonModules/android/general.py | 11 ++++++++ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 77c7aa12da..763ba3c15f 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): """ @@ -32,28 +34,30 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) - self.INCOMING_MSG_STRING = "Incoming" - self.OUTGOING_MSG_STRING = "Outgoing" self._DEFAULT_STRING = "" - self._DEFAULT_LONG = -1L + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = None + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - def get_account_name(self): - return self._DEFAULT_STRING + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO def get_call_direction(self): - return self._DEFAULT_STRING + return self._DEFAULT_DIRECTION def get_phone_number_from(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_call_start_date_time(self): return self._DEFAULT_LONG def get_call_end_date_time(self): return self._DEFAULT_LONG - - def get_contact_name(self): - return self._DEFAULT_STRING + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 0346a203e7..15c4166db7 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper class TskMessagesParser(ResultSetIterator): """ @@ -31,32 +33,34 @@ class TskMessagesParser(ResultSetIterator): def __init__(self, result_set): super(TskMessagesParser, self).__init__(result_set) - self.INCOMING_MSG_STRING = "Incoming" - self.OUTGOING_MSG_STRING = "Outgoing" self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_INT = -1 + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = None + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN - def get_account_id(self): - return self._DEFAULT_TEXT + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD def get_message_type(self): return self._DEFAULT_TEXT def get_message_direction(self): - return self._DEFAULT_TEXT + return self._DEFAULT_COMMUNICATION_DIRECTION def get_phone_number_from(self): - return self._DEFAULT_TEXT + return self._DEFAULT_ACCOUNT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_TEXT + return self._DEFAULT_ACCOUNT_ADDRESS def get_message_date_time(self): return self._DEFAULT_LONG def get_message_read_status(self): - return self._DEFAULT_INT + return self._DEFAULT_MSG_READ_STATUS def get_message_subject(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body From 8259882b71b84518f60de7bc48ca87fbb19e5082 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 22:04:27 -0400 Subject: [PATCH 24/55] Fully implemented line parser --- .../android/TskCallLogsParser.py | 1 + InternalPythonModules/android/line.py | 211 +++++++++++------- 2 files changed, 136 insertions(+), 76 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 763ba3c15f..8c61070693 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -38,6 +38,7 @@ class TskCallLogsParser(ResultSetIterator): self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1 self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index df8db5d088..d7cab1b852 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -31,8 +31,8 @@ from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil -from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile @@ -57,18 +57,22 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) self._LINE_PACKAGE_NAME = "jp.naver.line.android" self._PARSER_NAME = "Line Parser" + self._VERSION = "9.15.1" def analyze(self, dataSource, fileManager, context): try: - contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line.db", True, self._LINE_PACKAGE_NAME) - calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", True, self._LINE_PACKAGE_NAME) + contact_and_message_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "naver_line", True, self._LINE_PACKAGE_NAME) + calllog_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "call_history", True, self._LINE_PACKAGE_NAME) for contact_and_message_db in contact_and_message_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) + helper = AppDBParserHelper(self._PARSER_NAME, + contact_and_message_db.getDBFile(), Account.Type.LINE) contacts_parser = LineContactsParser(contact_and_message_db) while contacts_parser.next(): - blackboard_util.addContact( + helper.addContact( contacts_parser.get_account_name(), contacts_parser.get_contact_name(), contacts_parser.get_phone(), @@ -77,11 +81,10 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): contacts_parser.get_email() ) contacts_parser.close() - """ - messages_parser = LineMessagesParser(line_db) + + messages_parser = LineMessagesParser(contact_and_message_db) while messages_parser.next(): - blackboard_util.addMessage( - messages_parser.get_account_id(), + helper.addMessage( messages_parser.get_message_type(), messages_parser.get_message_direction(), messages_parser.get_phone_number_from(), @@ -93,36 +96,27 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - """ contact_and_message_db.close() for calllog_db in calllog_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) - calllog_db.attachDatabase(dataSource, "naver_line.db", True, calllog_db.getDBFile().getParentPath(), "naver") + helper = AppDBParserHelper(self._PARSER_NAME, + calllog_db.getDBFile(), Account.Type.LINE) + calllog_db.attachDatabase(dataSource, + "naver_line", calllog_db.getDBFile().getParentPath(), "naver") calllog_parser = LineCallLogsParser(calllog_db) while calllog_parser.next(): - print(calllog_parser.get_account_name()) - print(calllog_parser.get_contact_name()) - print(calllog_parser.get_call_direction()) - print(calllog_parser.get_phone_number_from()) - print(calllog_parser.get_phone_number_to()) - print(calllog_parser.get_call_start_date_time()) - print(calllog_parser.get_call_end_date_time()) - blackboard_util.addCalllog( - calllog_parser.get_account_name(), + helper.addCalllog( calllog_parser.get_call_direction(), calllog_parser.get_phone_number_from(), calllog_parser.get_phone_number_to(), calllog_parser.get_call_start_date_time(), calllog_parser.get_call_end_date_time(), - calllog_parser.get_contact_name() + calllog_parser.get_call_type() ) - - calllog_db.detachDatabase("naver") calllog_parser.close() - calllog_db.close() + calllog_db.close() except (SQLException, TskCoreException) as ex: # Error parsing Line databases. self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) @@ -141,47 +135,62 @@ class LineCallLogsParser(TskCallLogsParser): SELECT substr(CallH.call_type, -1) AS direction, CallH.start_time AS start_time, CallH.end_time AS end_time, - ConT.server_name AS account_name, - ConT.name AS contact_name - FROM call_history AS CallH - JOIN naver.contacts AS ConT - ON CallH.caller_mid = ConT.m_id + ConT.server_name AS name, + CallH.voip_type AS call_type, + ConT.m_id + FROM call_history AS CallH + JOIN naver.contacts AS ConT + ON CallH.caller_mid = ConT.m_id """ ) ) - self._OUTGOING_CALL = "O" - self._INCOMING_CALL = "I" + self._OUTGOING_CALL_TYPE = "O" + self._INCOMING_CALL_TYPE = "I" self._had_error = False + self._VIDEO_CALL_TYPE = "V" + self._AUDIO_CALL_TYPE = "A" - def get_call_direction(self): - direction = self.result_set.getString("direction") - if direction == self._OUTGOING_CALL: - return self.OUTGOING_MSG_STRING - return self.INCOMING_MSG_STRING + def get_call_direction(self): + direction = self.result_set.getString("direction") + if direction == self._OUTGOING_CALL_TYPE: + return self.OUTGOING_CALL + return self.INCOMING_CALL - def get_call_start_date_time(self): - start_time = self.result_set.getString("start_time") - try: - return long(start_time) / 1000 - except ValueError as ve: - self._had_error = True - print("bad_conversion") - return super(LineCallLogsParser, self).get_call_start_date_time() + def get_call_start_date_time(self): + try: + return long(self.result_set.getString("start_time")) / 1000 + except ValueError as ve: + self._had_error = True + return super(LineCallLogsParser, self).get_call_start_date_time() - def get_call_end_date_time(self): - end_time = self.result_set.getString("end_time") - try: - return long(end_time) / 1000 - except ValueError as ve: - self._had_error = True - print("bad conversion") - return super(LineCallLogsParser, self).get_call_end_date_time() + def get_call_end_date_time(self): + try: + return long(self.result_set.getString("end_time")) / 1000 + except ValueError as ve: + self._had_error = True + return super(LineCallLogsParser, self).get_call_end_date_time() + + def get_phone_number_to(self): + if self.get_call_direction() == self.OUTGOING_CALL: + return Account.Address(self.result_set.getString("m_id"), + self.result_set.getString("name")) + return super(LineCallLogsParser, self).get_phone_number_to() - def get_account_name(self): - return self.result_set.getString("account_name") + def get_phone_number_from(self): + if self.get_call_direction() == self.INCOMING_CALL: + return Account.Address(self.result_set.getString("m_id"), + self.result_set.getString("name")) + return super(LineCallLogsParser, self).get_phone_number_from() - def get_contact_name(self): - return self.result_set.getString("contact_name") + def get_call_type(self): + if self.result_set.getString("call_type") == self._VIDEO_CALL_TYPE: + return self.VIDEO_CALL + if self.result_set.getString("call_type") == self._AUDIO_CALL_TYPE: + return self.AUDIO_CALL + return super(LineCallLogsParser, self).get_call_type() + + def has_incomplete_results(self): + return self._had_error class LineContactsParser(TskContactsParser): """ @@ -193,17 +202,17 @@ class LineContactsParser(TskContactsParser): def __init__(self, contact_db): super(LineContactsParser, self).__init__(contact_db.runQuery( """ - SELECT name, + SELECT m_id, server_name FROM contacts """ ) ) def get_account_name(self): - return self.result_set.getString("server_name") + return self.result_set.getString("m_id") def get_contact_name(self): - return self.result_set.getString("name") + return self.result_set.getString("server_name") class LineMessagesParser(TskMessagesParser): """ @@ -213,23 +222,45 @@ class LineMessagesParser(TskMessagesParser): """ def __init__(self, message_db): - super().__init__(message_db.runQuery( - """SELECT created_time, content, contacts.server_name AS server_name, read_count - FROM chat_history - JOIN contacts - ON chat_history.from_mid = contacts.m_id""" - )) + super(LineMessagesParser, self).__init__(message_db.runQuery( + """ + SELECT all_contacts.name, + all_contacts.id, + all_contacts.members, + CH.from_mid, + CH.content, + CH.created_time, + CH.attachement_type, + CH.attachement_local_uri, + CH.status + FROM (SELECT G.name, + group_members.id, + group_members.members + FROM (SELECT id, + group_concat(m_id) AS members + FROM membership + GROUP BY id) AS group_members + JOIN groups AS G + ON G.id = group_members.id + UNION + SELECT server_name, + m_id, + NULL + FROM contacts) AS all_contacts + JOIN chat_history AS CH + ON CH.chat_id = all_contacts.id + """ + ) + ) self._LINE_MESSAGE_TYPE = "Line Message" + #From the limited test data, it appeared that incoming + #was only associated with a 1 status. Status # 3 and 7 + #was only associated with outgoing. + self._INCOMING_MESSAGE_TYPE = 1 self._had_error = False - def get_account_id(self): - return self.result_set.getString("server_name") - def get_message_type(self): - return self.LINE_MESSAGE_TYPE - - def get_phone_number_from(self): - return self.result_set("server_name") + return self._LINE_MESSAGE_TYPE def get_message_date_time(self): created_time = self.result_set.getString("created_time") @@ -242,6 +273,34 @@ class LineMessagesParser(TskMessagesParser): def get_message_text(self): content = self.result_set.getString("content") - if not LineContentUtil.is_text_message(content): - return "" return content + + def get_message_direction(self): + if self.result_set.getInt("status") == self._INCOMING_MESSAGE_TYPE: + return self.INCOMING + return self.OUTGOING + + def get_phone_number_from(self): + if self.get_message_direction() == self.INCOMING: + group = self.result_set.getString("members") + if group is None: + return Account.Address(self.result_set.getString("from_mid"), + self.result_set.getString("name")) + return Account.Address(self.result_set.getString("from_mid"), + self.result_set.getString("name")) + return super(LineMessagesParser, self).get_phone_number_from() + + def get_phone_number_to(self): + if self.get_message_direction() == self.OUTGOING: + group = self.result_set.getString("members") + if group is None: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("name")) + return Account.Address(group, self.result_set.getString("name")) + return super(LineMessagesParser, self).get_phone_number_to() + + def get_thread_id(self): + members = self.result_set.getString("members") + if members is not None: + return self.result_set.getString("id") + return super(LineMessagesParser, self).get_thread_id() From 5b87544b97fe22b6bae72f8eb1c1aca52ea9c993 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 23:29:10 -0400 Subject: [PATCH 25/55] Added attachments and removed call logs --- InternalPythonModules/android/line.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index d7cab1b852..1dc9871329 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -44,6 +44,7 @@ from org.sleuthkit.datamodel import Account from TskContactsParser import TskContactsParser from TskMessagesParser import TskMessagesParser from TskCallLogsParser import TskCallLogsParser +from general import appendAttachmentList import traceback import general @@ -114,6 +115,7 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): calllog_parser.get_call_end_date_time(), calllog_parser.get_call_type() ) + calllog_db.detachDatabase("naver") calllog_parser.close() calllog_db.close() @@ -249,6 +251,7 @@ class LineMessagesParser(TskMessagesParser): FROM contacts) AS all_contacts JOIN chat_history AS CH ON CH.chat_id = all_contacts.id + WHERE attachement_type != 6 """ ) ) @@ -273,6 +276,11 @@ class LineMessagesParser(TskMessagesParser): def get_message_text(self): content = self.result_set.getString("content") + attachment_uri = self.result_set.getString("attachement_local_uri") + if attachment_uri is not None and content is not None: + return appendAttachmentList(content, [attachment_uri]) + elif attachment_uri is not None and content is None: + return appendAttachmentList("", [attachment_uri]) return content def get_message_direction(self): From 296f18ce3c0f19ccc6eca402ecf399cce82edafe Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 23:33:46 -0400 Subject: [PATCH 26/55] initial infra commit --- .../android/ResultSetIterator.py | 35 +++++++++ .../android/TskCallLogsParser.py | 64 +++++++++++++++++ .../android/TskContactsParser.py | 49 +++++++++++++ .../android/TskMessagesParser.py | 72 +++++++++++++++++++ InternalPythonModules/android/general.py | 11 +++ 5 files changed, 231 insertions(+) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..8c61070693 --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,64 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self._DEFAULT_STRING = "" + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = None + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1 + + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + + def get_call_direction(self): + return self._DEFAULT_DIRECTION + + def get_phone_number_from(self): + return self._DEFAULT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ADDRESS + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..15c4166db7 --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,72 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = None + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_COMMUNICATION_DIRECTION + + def get_phone_number_from(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_MSG_READ_STATUS + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body From 8e95f1486253482a1bffe07de1cf03bde30ac011 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 23:43:21 -0400 Subject: [PATCH 27/55] Initial skeleton commit --- InternalPythonModules/android/skype.py | 194 +++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 InternalPythonModules/android/skype.py diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py new file mode 100644 index 0000000000..bbf1afccc3 --- /dev/null +++ b/InternalPythonModules/android/skype.py @@ -0,0 +1,194 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskMessagesParser import TskMessagesParser +from TskContactsParser import TskContactsParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class SkypeAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the Viber App databases for TSK contacts, message + and calllog artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._SKYPE_PACKAGE_NAME = "" + self._PARSER_NAME = "Skype Parser" + self._VERSION = "" + + def analyze(self, dataSource, fileManager, context): + """ + Extract, Transform and Load all messages, contacts and + calllogs from the Skype databases. + """ + + try: + skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "", True, self._SKYPE_PACKAGE_NAME) + + #Extract TSK_CONTACT and TSK_CALLLOG information + for skype_db in skype_dbs: + helper = AppDBParserHelper(self._PARSER_NAME, + contact_and_calllog_db.getDBFile(), Account.Type.SKYPE) + + contacts_parser = SkypeContactsParser(skype_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + + calllog_parser = SkypeCallLogsParser(skype_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + + messages_parser = SkypeMessagesParser(skype_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + + skype_db.close() + except (SQLException, TskCoreException) as ex: + #Error parsing Viber db + self._logger.log(Level.WARNING, "Error parsing Skype Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + +class SkypeCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from the Skype database. + TSK_CALLLOG fields that are not in the Skype database are given + a default value inherited from the super class. + """ + + def __init__(self, calllog_db): + super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery( + """ + """ + ) + ) + + # def get_phone_number_from(self): + + # def get_phone_number_to(self): + + # def get_call_direction(self): + + # def get_call_start_date_time(self): + +# def get_call_end_date_time(self): + +# def get_call_type(self): + +class SkypeContactsParser(TskContactsParser): + """ + Extracts TSK_CONTACT information from the Skype database. + TSK_CONTACT fields that are not in the Skype database are given + a default value inherited from the super class. + """ + + def __init__(self, contact_db): + super(SkypeContactsParser, self).__init__(contact_db.runQuery( + """ + """ + ) + ) + +# def get_account_name(self): + +# def get_contact_name(self): + +# def get_phone(self): + +class SkypeMessagesParser(TskMessagesParser): + """ + Extract TSK_MESSAGE information from the Skype database. + TSK_CONTACT fields that are not in the Skype database are given + a default value inherited from the super class. + """ + + def __init__(self, message_db): + super(SkypeMessagesParser, self).__init__(message_db.runQuery( + """ + """ + ) + ) + + #def get_message_type(self): + + #def get_phone_number_from(self): + + #def get_message_direction(self): + + #def get_phone_number_to(self): + + #def get_message_date_time(self): + + #def get_message_read_status(self): + + #def get_message_text(self): + + #def get_thread_id(self): From 08ce5a4555852968b777b02210144250174cc980 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 16 Sep 2019 10:08:55 -0400 Subject: [PATCH 28/55] Implemented contacts and some of the call logs parsing --- InternalPythonModules/android/module.py | 3 +- InternalPythonModules/android/skype.py | 112 ++++++++++++++++++++---- 2 files changed, 96 insertions(+), 19 deletions(-) diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..796f4026da 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import skype class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), skype.SkypeAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index bbf1afccc3..a4db7de556 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -53,9 +53,9 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): def __init__(self): self._logger = Logger.getLogger(self.__class__.__name__) - self._SKYPE_PACKAGE_NAME = "" + self._SKYPE_PACKAGE_NAME = "com.skype.raider" self._PARSER_NAME = "Skype Parser" - self._VERSION = "" + self._VERSION = "8.15.0.428" def analyze(self, dataSource, fileManager, context): """ @@ -65,12 +65,12 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): try: skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "", True, self._SKYPE_PACKAGE_NAME) + "live", False, "") #self._SKYPE_PACKAGE_NAME) #Extract TSK_CONTACT and TSK_CALLLOG information for skype_db in skype_dbs: helper = AppDBParserHelper(self._PARSER_NAME, - contact_and_calllog_db.getDBFile(), Account.Type.SKYPE) + skype_db.getDBFile(), Account.Type.SKYPE) contacts_parser = SkypeContactsParser(skype_db) while contacts_parser.next(): @@ -83,7 +83,7 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): contacts_parser.get_email() ) contacts_parser.close() - + """ calllog_parser = SkypeCallLogsParser(skype_db) while calllog_parser.next(): helper.addCalllog( @@ -110,7 +110,7 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - + """ skype_db.close() except (SQLException, TskCoreException) as ex: #Error parsing Viber db @@ -127,21 +127,75 @@ class SkypeCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery( """ + SELECT full_contacts_list.id, + full_contacts_list.members, + full_contacts_list.names, + time, + duration, + is_sender_me, + person_id + FROM (SELECT conversation_id AS id , + Group_concat(person_id) AS members, + Group_concat(CASE + WHEN Ifnull(first_name, "") == "" + AND Ifnull(last_name,"") == "" THEN entry_id + WHEN Ifnull(first_name, "") == "" THEN last_name + WHEN Ifnull(last_name, "") == "" THEN first_name + ELSE first_name || " " || last_name + END) AS names + FROM particiapnt AS PART + JOIN person AS P + ON PART.person_id = P.entry_id + GROUP BY conversation_id + UNION + SELECT entry_id AS id, + NULL, + CASE + WHEN Ifnull(first_name, "") == "" + AND Ifnull(last_name, "") == "" THEN entry_id + WHEN Ifnull(first_name, "") == "" THEN last_name + WHEN Ifnull(last_name, "") == "" THEN first_name + ELSE first_name + || " " + || last_name + end AS name + FROM person) AS full_contacts_list + JOIN chatitem AS C + ON C.conversation_link = full_contacts_list.id + WHERE message_type == 3 """ ) ) + self._INCOMING_CALL_TYPE = 0 + self._OUTGOING_CALL_TYPE = 1 + - # def get_phone_number_from(self): + def get_phone_number_from(self): + if self.get_call_direction() == self._INCOMING_CALL_TYPE: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("names")) + return super(SkypeCallLogsParser, self).get_phone_number_from() - # def get_phone_number_to(self): + def get_phone_number_to(self): + if self.get_call_direction() == self._OUTGOING_CALL_TYPE: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("names")) + return super(SkypeCallLogsParser, self).get_phone_number_to() + - # def get_call_direction(self): + def get_call_direction(self): + direction = self.result_set.getInt("is_sender_me") + if direction == self._INCOMING_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL - # def get_call_start_date_time(self): + def get_call_start_date_time(self): + return self.result_set.getLong("time") / 1000 -# def get_call_end_date_time(self): - -# def get_call_type(self): + def get_call_end_date_time(self): + start = self.get_call_start_date_time() + duration = self.result_set.getInt("duration") / 1000 + return start + duration class SkypeContactsParser(TskContactsParser): """ @@ -153,15 +207,37 @@ class SkypeContactsParser(TskContactsParser): def __init__(self, contact_db): super(SkypeContactsParser, self).__init__(contact_db.runQuery( """ + SELECT entry_id, + CASE + WHEN Ifnull(first_name, "") == "" + AND Ifnull(last_name, "") == "" THEN entry_id + WHEN Ifnull(first_name, "") == "" THEN last_name + WHEN Ifnull(last_name, "") == "" THEN first_name + ELSE first_name + || " " + || last_name + end AS name + FROM person + union + SELECT entry_id, CASE + WHEN Ifnull(first_name, "") == "" + AND Ifnull(last_name, "") == "" THEN entry_id + WHEN Ifnull(first_name, "") == "" THEN last_name + WHEN Ifnull(last_name, "") == "" THEN first_name + ELSE first_name + || " " + || last_name + end AS name + FROM user """ - ) + ) ) -# def get_account_name(self): + def get_account_name(self): + return self.result_set.getString("entry_id") -# def get_contact_name(self): - -# def get_phone(self): + def get_contact_name(self): + return self.result_set.getString("name") class SkypeMessagesParser(TskMessagesParser): """ From 2b10527b33d4e8b35e6a484deb5121171240b8b9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 17 Sep 2019 13:35:18 -0400 Subject: [PATCH 29/55] fully implemented skype parser --- InternalPythonModules/android/skype.py | 162 +++++++++++++++++++------ 1 file changed, 127 insertions(+), 35 deletions(-) diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index a4db7de556..b88becf295 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -47,7 +47,7 @@ import general class SkypeAnalyzer(general.AndroidComponentAnalyzer): """ - Parses the Viber App databases for TSK contacts, message + Parses the Skype App databases for TSK contacts, message and calllog artifacts. """ @@ -56,6 +56,25 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): self._SKYPE_PACKAGE_NAME = "com.skype.raider" self._PARSER_NAME = "Skype Parser" self._VERSION = "8.15.0.428" + + def get_account_instance(self, skype_db): + account_query_result = skype_db.runQuery( + """ + SELECT entry_id, + CASE + WHEN first_name is NULL AND last_name is NULL THEN entry_id + WHEN first_name is NULL THEN last_name + WHEN last_name is NULL THEN first_name + ELSE first_name || " " || last_name + END as name + FROM user + """ + ) + + if account_query_result is not None and account_query_result.next(): + return Account.Address(account_query_result.getString("entry_id"), + account_query_result.getString("name")) + return None def analyze(self, dataSource, fileManager, context): """ @@ -64,13 +83,21 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): """ try: + #Skype databases are of the form: live:XYZ.db, where + #XYZ is the skype id of the user. skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, "live", False, "") #self._SKYPE_PACKAGE_NAME) #Extract TSK_CONTACT and TSK_CALLLOG information for skype_db in skype_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, - skype_db.getDBFile(), Account.Type.SKYPE) + account_instance = self.get_account_instance(skype_db) + if account_instance is None: + helper = AppDBParserHelper(self._PARSER_NAME, + skype_db.getDBFile(), Account.Type.SKYPE) + else: + helper = AppDBParserHelper(self._PARSER_NAME, + skype_db.getDBFile(), Account.Type.SKYPE, + Account.Type.SKYPE, account_instance) contacts_parser = SkypeContactsParser(skype_db) while contacts_parser.next(): @@ -83,7 +110,7 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): contacts_parser.get_email() ) contacts_parser.close() - """ + calllog_parser = SkypeCallLogsParser(skype_db) while calllog_parser.next(): helper.addCalllog( @@ -110,7 +137,7 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - """ + skype_db.close() except (SQLException, TskCoreException) as ex: #Error parsing Viber db @@ -128,12 +155,10 @@ class SkypeCallLogsParser(TskCallLogsParser): super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery( """ SELECT full_contacts_list.id, - full_contacts_list.members, full_contacts_list.names, time, duration, - is_sender_me, - person_id + is_sender_me FROM (SELECT conversation_id AS id , Group_concat(person_id) AS members, Group_concat(CASE @@ -141,7 +166,9 @@ class SkypeCallLogsParser(TskCallLogsParser): AND Ifnull(last_name,"") == "" THEN entry_id WHEN Ifnull(first_name, "") == "" THEN last_name WHEN Ifnull(last_name, "") == "" THEN first_name - ELSE first_name || " " || last_name + ELSE first_name + || " " + || last_name END) AS names FROM particiapnt AS PART JOIN person AS P @@ -158,7 +185,7 @@ class SkypeCallLogsParser(TskCallLogsParser): ELSE first_name || " " || last_name - end AS name + END AS name FROM person) AS full_contacts_list JOIN chatitem AS C ON C.conversation_link = full_contacts_list.id @@ -171,23 +198,24 @@ class SkypeCallLogsParser(TskCallLogsParser): def get_phone_number_from(self): - if self.get_call_direction() == self._INCOMING_CALL_TYPE: + if self.get_call_direction() == self.INCOMING_CALL: return Account.Address(self.result_set.getString("id"), self.result_set.getString("names")) return super(SkypeCallLogsParser, self).get_phone_number_from() def get_phone_number_to(self): - if self.get_call_direction() == self._OUTGOING_CALL_TYPE: + if self.get_call_direction() == self.OUTGOING_CALL: return Account.Address(self.result_set.getString("id"), self.result_set.getString("names")) return super(SkypeCallLogsParser, self).get_phone_number_to() - def get_call_direction(self): direction = self.result_set.getInt("is_sender_me") - if direction == self._INCOMING_TYPE: + if direction == self._INCOMING_CALL_TYPE: return self.INCOMING_CALL - return self.OUTGOING_CALL + if direction == self._OUTGOING_CALL_TYPE: + return self.OUTGOING_CALL + return super(SkypeCallLogsParser, self).get_call_direction() def get_call_start_date_time(self): return self.result_set.getLong("time") / 1000 @@ -218,17 +246,6 @@ class SkypeContactsParser(TskContactsParser): || last_name end AS name FROM person - union - SELECT entry_id, CASE - WHEN Ifnull(first_name, "") == "" - AND Ifnull(last_name, "") == "" THEN entry_id - WHEN Ifnull(first_name, "") == "" THEN last_name - WHEN Ifnull(last_name, "") == "" THEN first_name - ELSE first_name - || " " - || last_name - end AS name - FROM user """ ) ) @@ -249,22 +266,97 @@ class SkypeMessagesParser(TskMessagesParser): def __init__(self, message_db): super(SkypeMessagesParser, self).__init__(message_db.runQuery( """ - """ + SELECT full_contacts_list.id, + full_contacts_list.members, + full_contacts_list.names, + time, + content, + file_name, + device_gallery_path, + is_sender_me + FROM (SELECT conversation_id AS id , + Group_concat(person_id) AS members, + Group_concat(CASE + WHEN Ifnull(first_name, "") == "" + AND Ifnull(last_name,"") == "" THEN entry_id + WHEN Ifnull(first_name, "") == "" THEN last_name + WHEN Ifnull(last_name, "") == "" THEN first_name + ELSE first_name + || " " + || last_name + END) AS names + FROM particiapnt AS PART + JOIN person AS P + ON PART.person_id = P.entry_id + GROUP BY conversation_id + UNION + SELECT entry_id AS id, + NULL, + CASE + WHEN Ifnull(first_name, "") == "" + AND Ifnull(last_name, "") == "" THEN entry_id + WHEN Ifnull(first_name, "") == "" THEN last_name + WHEN Ifnull(last_name, "") == "" THEN first_name + ELSE first_name + || " " + || last_name + END AS name + FROM person) AS full_contacts_list + JOIN chatitem AS C + ON C.conversation_link = full_contacts_list.id + WHERE message_type != 3 + """ ) ) + self._SKYPE_MESSAGE_TYPE = "Skype Message" + self._OUTGOING_MESSAGE_TYPE = 1 + self._INCOMING_MESSAGE_TYPE = 0 - #def get_message_type(self): + def get_message_type(self): + return self._SKYPE_MESSAGE_TYPE - #def get_phone_number_from(self): + def get_phone_number_from(self): + if self.get_message_direction() == self.INCOMING: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("names")) + return super(SkypeMessagesParser, self).get_phone_number_from() - #def get_message_direction(self): + def get_message_direction(self): + direction = self.result_set.getInt("is_sender_me") + if direction == self._OUTGOING_MESSAGE_TYPE: + return self.OUTGOING + if direction == self._INCOMING_MESSAGE_TYPE: + return self.INCOMING + return super(SkypeMessagesParser, self).get_message_direction() - #def get_phone_number_to(self): + def get_phone_number_to(self): + if self.get_message_direction() == self.OUTGOING: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("names")) + return super(SkypeMessagesParser, self).get_phone_number_to() - #def get_message_date_time(self): + def get_message_date_time(self): + date = self.result_set.getLong("time") + return date / 1000 - #def get_message_read_status(self): + def get_message_text(self): + content = self.result_set.getString("content") - #def get_message_text(self): + if content is not None: + file_name = self.result_set.getString("file_name") + file_path = self.result_set.getString("device_gallery_path") - #def get_thread_id(self): + #if a file name and file path are associated with a message, append it + if file_name is not None and file_path is not None: + attachment = "File Name: "+file_name +"\n"+ "File Path: "+file_path + return general.appendAttachmentList(content, [attachment]) + + return content + + return super(SkypeMessagesParser, self).get_message_text() + + def get_thread_id(self): + members = self.result_set.getString("members") + if members is not None: + return self.result_set.getString("id") + return super(SkypeMessagesParser, self).get_thread_id() From 7a6be8dd86d1fbb301e8b5af45411387e431fbae Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 17 Sep 2019 17:30:18 -0400 Subject: [PATCH 30/55] Made incremental improvements --- InternalPythonModules/android/skype.py | 227 +++++++++++++++---------- 1 file changed, 136 insertions(+), 91 deletions(-) diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index b88becf295..4a236a7206 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -49,6 +49,29 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): """ Parses the Skype App databases for TSK contacts, message and calllog artifacts. + + About version 8.15.0.428 (9/17/2019) Skype database: + - There are 4 tables this parser uses: + 1) person - this table appears to hold all contacts known to the user. + 2) user - this table holds information pertaining to the user. + 3) particiapnt - Yes, that is not a typo. This table maps group chat + ids to skype ids (1 to many). + 4) chatItem - This table contains all messages. It maps the group id or + skype id (for 1 to 1 communication) to the message content + and metadata. Either the group id or skype id is stored in + a column named 'conversation_link'. + + More info and implementation details: + - The person table does not include groups. To get + all 1 to 1 communications, we could simply join the person and chatItem tables. + This would mean we'd need to do a second pass to get all the group information + as they would be excluded in the join. Since the chatItem table stores both the + group id or skype_id in one column, the person and particiapnt table are unioned + together so that all rows are matched in one join with chatItem. This result is + labeled contact_list_with_groups in the following queries. + - In order to keep the formatting of the name consistent throughout each query, + a _format_user_name() function was created to encapsulate the CASE statement + that was being shared across them. Refer to the method for more details. """ def __init__(self): @@ -60,13 +83,8 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): def get_account_instance(self, skype_db): account_query_result = skype_db.runQuery( """ - SELECT entry_id, - CASE - WHEN first_name is NULL AND last_name is NULL THEN entry_id - WHEN first_name is NULL THEN last_name - WHEN last_name is NULL THEN first_name - ELSE first_name || " " || last_name - END as name + SELECT entry_id, + """+_format_user_name()+""" AS name FROM user """ ) @@ -77,18 +95,12 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): return None def analyze(self, dataSource, fileManager, context): - """ - Extract, Transform and Load all messages, contacts and - calllogs from the Skype databases. - """ - try: #Skype databases are of the form: live:XYZ.db, where #XYZ is the skype id of the user. skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, "live", False, "") #self._SKYPE_PACKAGE_NAME) - #Extract TSK_CONTACT and TSK_CALLLOG information for skype_db in skype_dbs: account_instance = self.get_account_instance(skype_db) if account_instance is None: @@ -98,7 +110,9 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): helper = AppDBParserHelper(self._PARSER_NAME, skype_db.getDBFile(), Account.Type.SKYPE, Account.Type.SKYPE, account_instance) - + + #Query for contacts and iterate row by row adding + #each contact artifact contacts_parser = SkypeContactsParser(skype_db) while contacts_parser.next(): helper.addContact( @@ -111,6 +125,8 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): ) contacts_parser.close() + #Query for call logs and iterate row by row adding + #each call log artifact calllog_parser = SkypeCallLogsParser(skype_db) while calllog_parser.next(): helper.addCalllog( @@ -123,6 +139,8 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): ) calllog_parser.close() + #Query for messages and iterate row by row adding + #each message artifact messages_parser = SkypeMessagesParser(skype_db) while messages_parser.next(): helper.addMessage( @@ -140,7 +158,7 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): skype_db.close() except (SQLException, TskCoreException) as ex: - #Error parsing Viber db + #Error parsing Skype db self._logger.log(Level.WARNING, "Error parsing Skype Databases", ex) self._logger.log(Level.WARNING, traceback.format_exec()) @@ -154,41 +172,36 @@ class SkypeCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery( """ - SELECT full_contacts_list.id, - full_contacts_list.names, + SELECT contacts_list_with_groups.conversation_id, + contacts_list_with_groups.participant_ids, + contacts_list_with_groups.participants, time, duration, - is_sender_me - FROM (SELECT conversation_id AS id , - Group_concat(person_id) AS members, - Group_concat(CASE - WHEN Ifnull(first_name, "") == "" - AND Ifnull(last_name,"") == "" THEN entry_id - WHEN Ifnull(first_name, "") == "" THEN last_name - WHEN Ifnull(last_name, "") == "" THEN first_name - ELSE first_name - || " " - || last_name - END) AS names + is_sender_me, + person_id as sender_id, + sender_name.name as sender_name + FROM (SELECT conversation_id, + Group_concat(person_id) AS participant_ids, + Group_concat("""+_format_user_name()+""") AS participants FROM particiapnt AS PART JOIN person AS P ON PART.person_id = P.entry_id GROUP BY conversation_id UNION - SELECT entry_id AS id, - NULL, - CASE - WHEN Ifnull(first_name, "") == "" - AND Ifnull(last_name, "") == "" THEN entry_id - WHEN Ifnull(first_name, "") == "" THEN last_name - WHEN Ifnull(last_name, "") == "" THEN first_name - ELSE first_name - || " " - || last_name - END AS name - FROM person) AS full_contacts_list + SELECT entry_id, + NULL, + """+_format_user_name()+""" AS participant + FROM person) AS contacts_list_with_groups JOIN chatitem AS C - ON C.conversation_link = full_contacts_list.id + ON C.conversation_link = contacts_list_with_groups.conversation_id + JOIN (SELECT entry_id as id, + """+_format_user_name()+""" AS name + FROM person + UNION + SELECT entry_id as id, + """+_format_user_name()+""" AS name + FROM user) AS sender_name + ON sender_name.id = C.person_id WHERE message_type == 3 """ ) @@ -199,16 +212,28 @@ class SkypeCallLogsParser(TskCallLogsParser): def get_phone_number_from(self): if self.get_call_direction() == self.INCOMING_CALL: - return Account.Address(self.result_set.getString("id"), - self.result_set.getString("names")) - return super(SkypeCallLogsParser, self).get_phone_number_from() + return Account.Address(self.result_set.getString("sender_id"), + self.result_set.getString("sender_name")) def get_phone_number_to(self): if self.get_call_direction() == self.OUTGOING_CALL: - return Account.Address(self.result_set.getString("id"), - self.result_set.getString("names")) + group_ids = self.result_set.getString("participant_ids") + name = self.result_set.getString("participants") + + if group_ids is not None: + group_ids = group_ids.split(",") + name = name.split(",") + recipients = [] + + for person_id, person_name in zip(group_ids, name): + recipients.append(Account.Address(person_id, person_name)) + + return recipients + + return Account.Address(self.result_set.getString("conversation_id"), name) + return super(SkypeCallLogsParser, self).get_phone_number_to() - + def get_call_direction(self): direction = self.result_set.getInt("is_sender_me") if direction == self._INCOMING_CALL_TYPE: @@ -236,15 +261,7 @@ class SkypeContactsParser(TskContactsParser): super(SkypeContactsParser, self).__init__(contact_db.runQuery( """ SELECT entry_id, - CASE - WHEN Ifnull(first_name, "") == "" - AND Ifnull(last_name, "") == "" THEN entry_id - WHEN Ifnull(first_name, "") == "" THEN last_name - WHEN Ifnull(last_name, "") == "" THEN first_name - ELSE first_name - || " " - || last_name - end AS name + """+_format_user_name()+""" AS name FROM person """ ) @@ -266,44 +283,38 @@ class SkypeMessagesParser(TskMessagesParser): def __init__(self, message_db): super(SkypeMessagesParser, self).__init__(message_db.runQuery( """ - SELECT full_contacts_list.id, - full_contacts_list.members, - full_contacts_list.names, + SELECT contacts_list_with_groups.conversation_id, + contacts_list_with_groups.participant_ids, + contacts_list_with_groups.participants, time, content, file_name, device_gallery_path, - is_sender_me - FROM (SELECT conversation_id AS id , - Group_concat(person_id) AS members, - Group_concat(CASE - WHEN Ifnull(first_name, "") == "" - AND Ifnull(last_name,"") == "" THEN entry_id - WHEN Ifnull(first_name, "") == "" THEN last_name - WHEN Ifnull(last_name, "") == "" THEN first_name - ELSE first_name - || " " - || last_name - END) AS names + is_sender_me, + person_id as sender_id, + sender_name.name AS sender_name + FROM (SELECT conversation_id, + Group_concat(person_id) AS participant_ids, + Group_concat("""+_format_user_name()+""") AS participants FROM particiapnt AS PART JOIN person AS P ON PART.person_id = P.entry_id GROUP BY conversation_id UNION - SELECT entry_id AS id, + SELECT entry_id as conversation_id, NULL, - CASE - WHEN Ifnull(first_name, "") == "" - AND Ifnull(last_name, "") == "" THEN entry_id - WHEN Ifnull(first_name, "") == "" THEN last_name - WHEN Ifnull(last_name, "") == "" THEN first_name - ELSE first_name - || " " - || last_name - END AS name - FROM person) AS full_contacts_list + """+_format_user_name()+""" AS participant + FROM person) AS contacts_list_with_groups JOIN chatitem AS C - ON C.conversation_link = full_contacts_list.id + ON C.conversation_link = contacts_list_with_groups.conversation_id + JOIN (SELECT entry_id as id, + """+_format_user_name()+""" AS name + FROM person + UNION + SELECT entry_id as id, + """+_format_user_name()+""" AS name + FROM user) AS sender_name + ON sender_name.id = C.person_id WHERE message_type != 3 """ ) @@ -317,8 +328,8 @@ class SkypeMessagesParser(TskMessagesParser): def get_phone_number_from(self): if self.get_message_direction() == self.INCOMING: - return Account.Address(self.result_set.getString("id"), - self.result_set.getString("names")) + return Account.Address(self.result_set.getString("sender_id"), + self.result_set.getString("sender_name")) return super(SkypeMessagesParser, self).get_phone_number_from() def get_message_direction(self): @@ -331,8 +342,21 @@ class SkypeMessagesParser(TskMessagesParser): def get_phone_number_to(self): if self.get_message_direction() == self.OUTGOING: - return Account.Address(self.result_set.getString("id"), - self.result_set.getString("names")) + group_ids = self.result_set.getString("participant_ids") + names = self.result_set.getString("participants") + + if group_ids is not None: + group_ids = group_ids.split(",") + names = names.split(",") + recipients = [] + + for participant_id, participant_name in zip(group_ids, names): + recipients.append(Account.Address(participant_id, participant_name)) + + return recipients + + return Account.Address(self.result_set.getString("conversation_id"), names) + return super(SkypeMessagesParser, self).get_phone_number_to() def get_message_date_time(self): @@ -356,7 +380,28 @@ class SkypeMessagesParser(TskMessagesParser): return super(SkypeMessagesParser, self).get_message_text() def get_thread_id(self): - members = self.result_set.getString("members") - if members is not None: - return self.result_set.getString("id") + group_ids = self.result_set.getString("participant_ids") + if group_ids is not None: + return self.result_set.getString("conversation_id") return super(SkypeMessagesParser, self).get_thread_id() + +def _format_user_name(): + """ + This CASE SQL statement is used in many queries to + format the names of users. For a user, there is a first_name + column and a last_name column. Some of these columns can be null + and our goal is to produce the cleanest data possible. In the event + that both the first and last name columns are null, we return the skype_id + which is stored in the database as 'entry_id'. + """ + + return """ + CASE + WHEN Ifnull(first_name, "") == "" AND Ifnull(last_name, "") == "" THEN entry_id + WHEN first_name is NULL THEN replace(last_name, ",", "") + WHEN last_name is NULL THEN replace(first_name, ",", "") + ELSE replace(first_name, ",", "") || " " || replace(last_name, ",", "") + END + """ + + From 65827284f1ec75fa9fb8e640ed09c354143c7c37 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 18 Sep 2019 16:05:33 -0400 Subject: [PATCH 31/55] Implement change --- .../dsp/AddLogicalImageTask.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 6c0bc5a0d6..50f6819264 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -63,6 +63,8 @@ final class AddLogicalImageTask implements Runnable { private final static String MODULE_NAME = "Logical Imager"; //NON-NLS private final static String ROOT_STR = "root"; // NON-NLS private final static String VHD_EXTENSION = ".vhd"; // NON-NLS + private final static int REPORT_PROGRESS_INTERVAL = 100; + private final static int POST_ARTIFACT_INTERVAL = 1000; private final String deviceId; private final String timeZone; private final File src; @@ -362,9 +364,12 @@ final class AddLogicalImageTask implements Runnable { String filename = fields[7]; String parentPath = fields[8]; - if (lineNumber % 100 == 0) { + if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); } + if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { + postArtifacts(artifacts); + } String query = makeQuery(createVHD, vhdFilename, fileMetaAddressStr, parentPath, filename); // TODO - findAllFilesWhere should SQL-escape the query @@ -375,15 +380,19 @@ final class AddLogicalImageTask implements Runnable { lineNumber++; } // end reading file - try { - // index the artifact for keyword search - blackboard.postArtifacts(artifacts, MODULE_NAME); - } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS - } + postArtifacts(artifacts); } } + private void postArtifacts(List artifacts) { + try { + // index the artifact for keyword search + blackboard.postArtifacts(artifacts, MODULE_NAME); + } catch (Blackboard.BlackboardException ex) { + LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS + } + } + private void addInterestingFileToArtifacts(AbstractFile file, String ruleSetName, String ruleName, List artifacts) throws TskCoreException { Collection attributes = new ArrayList<>(); BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, ruleSetName); @@ -440,7 +449,7 @@ final class AddLogicalImageTask implements Runnable { String ctime = fields[13]; parentPath = ROOT_STR + "/" + vhdFilename + "/" + parentPath; - if (lineNumber % 100 == 0) { + if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingExtractedFile(lineNumber, totalFiles)); } From a51d718645d4fbc94c4680623b0cd6c50a66f517 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 18 Sep 2019 16:08:44 -0400 Subject: [PATCH 32/55] clear artifacts after posting --- .../sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 50f6819264..64000e8f36 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -369,6 +369,7 @@ final class AddLogicalImageTask implements Runnable { } if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { postArtifacts(artifacts); + artifacts.clear(); } String query = makeQuery(createVHD, vhdFilename, fileMetaAddressStr, parentPath, filename); From e7b6e1a165896093646dd11b5fdf3a7aa932ee9e Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 19 Sep 2019 12:46:54 -0400 Subject: [PATCH 33/55] fix merge error --- .../dsp/AddLogicalImageTask.java | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index a569f93b75..b6ad885c4a 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -367,21 +367,19 @@ final class AddLogicalImageTask implements Runnable { List fileIds = entry.getValue(); for (Long fileId: fileIds) { - if (lineNumber % 100 == 0) { + if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); } + if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { + postArtifacts(artifacts); + artifacts.clear(); + } addInterestingFileToArtifacts(fileId, ruleSetName, ruleName, artifacts); lineNumber++; } iterator.remove(); } - - try { - // index the artifact for keyword search - blackboard.postArtifacts(artifacts, MODULE_NAME); - } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS - } + postArtifacts(artifacts); } private void addInterestingFileToArtifacts(long fileId, String ruleSetName, String ruleName, List artifacts) throws TskCoreException { @@ -429,11 +427,7 @@ final class AddLogicalImageTask implements Runnable { String parentPath = fields[8]; if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); - } - if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { - postArtifacts(artifacts); - artifacts.clear(); + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_searchingInterestingFile(lineNumber, totalFiles)); } String query = makeQuery(vhdFilename, fileMetaAddressStr, parentPath, filename); @@ -450,8 +444,6 @@ final class AddLogicalImageTask implements Runnable { } lineNumber++; } // end reading file - - postArtifacts(artifacts); } return interestingFileMap; } From 911995f489ad8c660e6738737efb4383e7f3a859 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 13:23:08 -0400 Subject: [PATCH 34/55] Updated the skype parser to use the new api --- .../android/TskCallLogsParser.py | 17 ++--- .../android/TskMessagesParser.py | 17 ++--- InternalPythonModules/android/skype.py | 70 ++++++++++++++----- 3 files changed, 69 insertions(+), 35 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 8c61070693..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,15 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - self._DEFAULT_LONG = -1 + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index 4a236a7206..8444d401cb 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -26,18 +26,25 @@ from java.sql import ResultSet from java.sql import SQLException from java.sql import Statement from java.util.logging import Level +from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser @@ -66,9 +73,10 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): all 1 to 1 communications, we could simply join the person and chatItem tables. This would mean we'd need to do a second pass to get all the group information as they would be excluded in the join. Since the chatItem table stores both the - group id or skype_id in one column, the person and particiapnt table are unioned - together so that all rows are matched in one join with chatItem. This result is - labeled contact_list_with_groups in the following queries. + group id or skype_id in one column, an implementation decision was made to union + the person and particiapnt table together so that all rows are matched in one join + with chatItem. This result is consistently labeled contact_list_with_groups in the + following queries. - In order to keep the formatting of the name consistent throughout each query, a _format_user_name() function was created to encapsulate the CASE statement that was being shared across them. Refer to the method for more details. @@ -95,19 +103,18 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): return None def analyze(self, dataSource, fileManager, context): - try: - #Skype databases are of the form: live:XYZ.db, where - #XYZ is the skype id of the user. - skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "live", False, "") #self._SKYPE_PACKAGE_NAME) - - for skype_db in skype_dbs: + #Skype databases are of the form: live:XYZ.db, where + #XYZ is the skype id of the user. + skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "live", False, self._SKYPE_PACKAGE_NAME) + for skype_db in skype_dbs: + try: account_instance = self.get_account_instance(skype_db) if account_instance is None: - helper = AppDBParserHelper(self._PARSER_NAME, + helper = CommunicationArtifactsHelper(self._PARSER_NAME, skype_db.getDBFile(), Account.Type.SKYPE) else: - helper = AppDBParserHelper(self._PARSER_NAME, + helper = CommunicationArtifactsHelper(self._PARSER_NAME, skype_db.getDBFile(), Account.Type.SKYPE, Account.Type.SKYPE, account_instance) @@ -155,12 +162,17 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - + except SQLException as ex: + #Error parsing Skype db + self._logger.log(Level.WARNING, "Error parsing Skype Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except (TskCoreException, BlackboardException) as ex: + #Severe error trying to add to case database.. case is not complete. + self._logger.log(Level.SEVERE, "Failed to add message artifacts" + + " to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + finally: skype_db.close() - except (SQLException, TskCoreException) as ex: - #Error parsing Skype db - self._logger.log(Level.WARNING, "Error parsing Skype Databases", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) class SkypeCallLogsParser(TskCallLogsParser): """ @@ -170,6 +182,20 @@ class SkypeCallLogsParser(TskCallLogsParser): """ def __init__(self, calllog_db): + """ + Big picture: + The query below creates a contacts_list_with_groups table, which + represents the recipient info. A chatItem record holds ids for + both the recipient and sender. The first join onto chatItem fills + in the blanks for the recipients. The second join back onto person + handles the sender info. The result is a table with all of the + communication details. + + Implementation details: + - message_type w/ value 3 appeared to be the call type, regardless + of if it was audio or video. + + """ super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery( """ SELECT contacts_list_with_groups.conversation_id, @@ -281,6 +307,11 @@ class SkypeMessagesParser(TskMessagesParser): """ def __init__(self, message_db): + """ + This query is very similar to the call logs query, the only difference is + it grabs more columns in the SELECT and excludes message_types which have + the call type value (3). + """ super(SkypeMessagesParser, self).__init__(message_db.runQuery( """ SELECT contacts_list_with_groups.conversation_id, @@ -392,7 +423,8 @@ def _format_user_name(): column and a last_name column. Some of these columns can be null and our goal is to produce the cleanest data possible. In the event that both the first and last name columns are null, we return the skype_id - which is stored in the database as 'entry_id'. + which is stored in the database as 'entry_id'. Commas are removed from the name + so that we can concatenate names into a comma seperate list for group chats. """ return """ From ab8860a63e8f23eb53cff19fcdeeddf79889c1b2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 14:26:26 -0400 Subject: [PATCH 35/55] updated skype and tested it --- InternalPythonModules/android/skype.py | 46 +++++++++++++++++++------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index 8444d401cb..ef7c0df0d5 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -41,6 +41,7 @@ from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus @@ -88,7 +89,7 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): self._PARSER_NAME = "Skype Parser" self._VERSION = "8.15.0.428" - def get_account_instance(self, skype_db): + def get_user_account(self, skype_db): account_query_result = skype_db.runQuery( """ SELECT entry_id, @@ -104,19 +105,36 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): #Skype databases are of the form: live:XYZ.db, where - #XYZ is the skype id of the user. + #XYZ is the skype id of the user. The following search + #does a generic substring match for 'live' in the skype + #package. skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "live", False, self._SKYPE_PACKAGE_NAME) + "live:", False, self._SKYPE_PACKAGE_NAME) + for skype_db in skype_dbs: try: - account_instance = self.get_account_instance(skype_db) - if account_instance is None: - helper = CommunicationArtifactsHelper(self._PARSER_NAME, - skype_db.getDBFile(), Account.Type.SKYPE) + #Attempt to get the user account id from the database + user_account_instance = None + try: + user_account_instance = self.get_user_account(skype_db) + except SQLException as ex: + self._logger.log(Level.WARNING, + "Error query for the user account in the Skype db.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + current_case = Case.getCurrentCaseThrows() + + if user_account_instance is None: + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + skype_db.getDBFile(), Account.Type.SKYPE + ) else: - helper = CommunicationArtifactsHelper(self._PARSER_NAME, - skype_db.getDBFile(), Account.Type.SKYPE, - Account.Type.SKYPE, account_instance) + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + skype_db.getDBFile(), Account.Type.SKYPE, + Account.Type.SKYPE, user_account_instance + ) #Query for contacts and iterate row by row adding #each contact artifact @@ -168,9 +186,13 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): self._logger.log(Level.WARNING, traceback.format_exc()) except (TskCoreException, BlackboardException) as ex: #Severe error trying to add to case database.. case is not complete. - self._logger.log(Level.SEVERE, "Failed to add message artifacts" + - " to the case database.", ex) + #These exceptions are thrown by the CommunicationArtifactsHelper. + self._logger.log(Level.SEVERE, + "Failed to add message artifacts to the case database.", ex) self._logger.log(Level.SEVERE, traceback.format_exc()) + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) finally: skype_db.close() From e979034b519cc523d908cac598b8d9d2598e9d0d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 14:33:02 -0400 Subject: [PATCH 36/55] Update base parser classes with new infra changes --- .../android/TskCallLogsParser.py | 16 +++++++++------- .../android/TskMessagesParser.py | 17 +++++++++-------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 763ba3c15f..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,14 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT From b052e9274df92b968f841b288b5fa9ec941de237 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 19 Sep 2019 15:21:00 -0400 Subject: [PATCH 37/55] Check for cancel --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index b6ad885c4a..f5d728b633 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -367,6 +367,10 @@ final class AddLogicalImageTask implements Runnable { List fileIds = entry.getValue(); for (Long fileId: fileIds) { + if (cancelled) { + postArtifacts(artifacts); + return; + } if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); } From 424dad7e148466286bc06b0ccd449b048cf37e92 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 15:53:04 -0400 Subject: [PATCH 38/55] made changes to textnow with updated infra --- InternalPythonModules/android/textnow.py | 198 +++++++++++++++-------- 1 file changed, 131 insertions(+), 67 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index 6ab7d14801..1471f720a5 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -26,22 +26,30 @@ from java.sql import ResultSet from java.sql import SQLException from java.sql import Statement from java.util.logging import Level +from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser -from general import appendAttachmentList import traceback import general @@ -50,8 +58,17 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): """ Parses the TextNow App databases for TSK contacts, message and calllog artifacts. - """ - + + The TextNow database in v6.41.0.2 is structured as follows: + - A messages table, which stores messages from/to a number + - A contacts table, which stores phone numbers + - A groups table, which stores each group the device owner is a part of + - A group_members table, which stores who is in each group + + The messages table contains both call logs and messages, with a type + column differentiating the two. + """ + def __init__(self): self._logger = Logger.getLogger(self.__class__.__name__) self._TEXTNOW_PACKAGE_NAME = "com.enflick.android.TextNow" @@ -64,61 +81,116 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): calllogs from the TextNow databases. """ - try: - textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, + textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, "textnow_data.db", True, self._TEXTNOW_PACKAGE_NAME) - for textnow_db in textnow_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, - textnow_db.getDBFile(), Account.Type.TEXTNOW) - - #Extract TSK_CONTACT information - contacts_parser = TextNowContactsParser(textnow_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() - - #Extract TSK_CALLLOG information - calllog_parser = TextNowCallLogsParser(textnow_db) - while calllog_parser.next(): - helper.addCalllog( - calllog_parser.get_call_direction(), - calllog_parser.get_phone_number_from(), - calllog_parser.get_phone_number_to(), - calllog_parser.get_call_start_date_time(), - calllog_parser.get_call_end_date_time(), - calllog_parser.get_call_type() - ) - calllog_parser.close() - - #Extract TSK_MESSAGES information - messages_parser = TextNowMessagesParser(textnow_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() - + for textnow_db in textnow_dbs: + try: + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + textnow_db.getDBFile(), Account.Type.TEXTNOW + ) + self.parse_contacts(textnow_db, helper) + self.parse_calllogs(textnow_db, helper) + self.parse_messages(textnow_db, helper) + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + finally: textnow_db.close() - except (SQLException, TskCoreException) as ex: + + def parse_contacts(self, textnow_db, helper): + #Query for contacts and iterate row by row adding + #each contact artifact + try: + contacts_parser = TextNowContactsParser(textnow_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: #Error parsing TextNow db - self._logger.log(Level.WARNING, "Error parsing TextNow Databases", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, "Error parsing TextNow databases for contacts", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifacts to the case database.. case database is not complete. + self._logger.log(Level.SEVERE, + "Error adding TextNow contacts artifacts to the case database", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard... + self._logger.log(Level.WARNING, + "Error posting TextNow contacts artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_calllogs(self, textnow_db, helper): + #Query for call logs and iterate row by row adding + #each call log artifact + try: + calllog_parser = TextNowCallLogsParser(textnow_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing TextNow databases for calllogs", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifacts to the case database.. case database is not complete. + self._logger.log(Level.SEVERE, + "Error adding TextNow call log artifacts to the case database", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard... + self._logger.log(Level.WARNING, + "Error posting TextNow call log artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_messages(self, textnow_db, helper): + #Query for messages and iterate row by row adding + #each message artifact + try: + messages_parser = TextNowMessagesParser(textnow_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + #Error parsing TextNow db + self._logger.log(Level.WARNING, "Error parsing TextNow databases for messages.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifacts to the case database.. case database is not complete. + self._logger.log(Level.SEVERE, + "Error adding TextNow messages artifacts to the case database", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard... + self._logger.log(Level.WARNING, + "Error posting TextNow messages artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) class TextNowCallLogsParser(TskCallLogsParser): """ @@ -128,6 +200,9 @@ class TextNowCallLogsParser(TskCallLogsParser): """ def __init__(self, calllog_db): + """ + message_type of 100 or 102 are for calls (audio, video) + """ super(TextNowCallLogsParser, self).__init__(calllog_db.runQuery( """ SELECT contact_value AS num, @@ -141,7 +216,6 @@ class TextNowCallLogsParser(TskCallLogsParser): ) self._INCOMING_CALL_TYPE = 1 self._OUTGOING_CALL_TYPE = 2 - self._has_errors = False def get_phone_number_from(self): if self.get_call_direction() == self.OUTGOING_CALL: @@ -169,7 +243,6 @@ class TextNowCallLogsParser(TskCallLogsParser): try: return start + long(duration) except ValueError as ve: - self._has_errors = True return super(TextNowCallLogsParser, self).get_call_end_date_time() class TextNowContactsParser(TskContactsParser): @@ -211,15 +284,6 @@ class TextNowMessagesParser(TskMessagesParser): def __init__(self, message_db): """ - The TextNow database in v6.41.0.2 is structured as follows: - - A messages table, which stores messages from/to a number - - A contacts table, which stores phone numbers - - A groups table, which stores each group the device owner is a part of - - A group_members table, which stores who is in each group - - The messages table contains both call logs and messages, with a type - column differentiating the two. - The query below does the following: - The group_info inner query creates a comma seperated list of group recipients for each group. This result is then joined on the groups table to get the thread id. @@ -311,7 +375,7 @@ class TextNowMessagesParser(TskMessagesParser): text = self.result_set.getString("message_text") attachment = self.result_set.getString("attach") if attachment != "": - text = appendAttachmentList(text, [attachment]) + text = general.appendAttachmentList(text, [attachment]) return text def get_thread_id(self): From 0c98287e83d8fcd22ec51f567923c9f7b5b12519 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 16:10:31 -0400 Subject: [PATCH 39/55] Updated skype to be more fault tolerant --- InternalPythonModules/android/skype.py | 159 ++++++++++++++++--------- 1 file changed, 103 insertions(+), 56 deletions(-) diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index ef7c0df0d5..23c8114c87 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -119,7 +119,7 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): user_account_instance = self.get_user_account(skype_db) except SQLException as ex: self._logger.log(Level.WARNING, - "Error query for the user account in the Skype db.", ex) + "Error querying for the user account in the Skype db.", ex) self._logger.log(Level.WARNING, traceback.format_exc()) current_case = Case.getCurrentCaseThrows() @@ -135,67 +135,114 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): skype_db.getDBFile(), Account.Type.SKYPE, Account.Type.SKYPE, user_account_instance ) - - #Query for contacts and iterate row by row adding - #each contact artifact - contacts_parser = SkypeContactsParser(skype_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() - - #Query for call logs and iterate row by row adding - #each call log artifact - calllog_parser = SkypeCallLogsParser(skype_db) - while calllog_parser.next(): - helper.addCalllog( - calllog_parser.get_call_direction(), - calllog_parser.get_phone_number_from(), - calllog_parser.get_phone_number_to(), - calllog_parser.get_call_start_date_time(), - calllog_parser.get_call_end_date_time(), - calllog_parser.get_call_type() - ) - calllog_parser.close() - - #Query for messages and iterate row by row adding - #each message artifact - messages_parser = SkypeMessagesParser(skype_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() - except SQLException as ex: - #Error parsing Skype db - self._logger.log(Level.WARNING, "Error parsing Skype Databases", ex) - self._logger.log(Level.WARNING, traceback.format_exc()) - except (TskCoreException, BlackboardException) as ex: - #Severe error trying to add to case database.. case is not complete. - #These exceptions are thrown by the CommunicationArtifactsHelper. - self._logger.log(Level.SEVERE, - "Failed to add message artifacts to the case database.", ex) - self._logger.log(Level.SEVERE, traceback.format_exc()) + self.parse_contacts(skype_db, helper) + self.parse_calllogs(skype_db, helper) + self.parse_messages(skype_db, helper) except NoCurrentCaseException as ex: self._logger.log(Level.WARNING, "No case currently open.", ex) self._logger.log(Level.WARNING, traceback.format_exc()) finally: skype_db.close() + def parse_contacts(self, skype_db, helper): + #Query for contacts and iterate row by row adding + #each contact artifact + try: + contacts_parser = SkypeContactsParser(skype_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: + #Error parsing Skype db + self._logger.log(Level.WARNING, + "Error parsing contact database for call logs artifacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Severe error trying to add to case database.. case is not complete. + #These exceptions are thrown by the CommunicationArtifactsHelper. + self._logger.log(Level.SEVERE, + "Failed to add contact artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Failed to post notification to blackboard + self._logger.log(Level.WARNING, + "Failed to post contact artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_calllogs(self, skype_db, helper): + #Query for call logs and iterate row by row adding + #each call log artifact + try: + calllog_parser = SkypeCallLogsParser(skype_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + except SQLException as ex: + #Error parsing Skype db + self._logger.log(Level.WARNING, + "Error parsing Skype database for call logs artifacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Severe error trying to add to case database.. case is not complete. + #These exceptions are thrown by the CommunicationArtifactsHelper. + self._logger.log(Level.SEVERE, + "Failed to add call log artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Failed to post notification to blackboard + self._logger.log(Level.WARNING, + "Failed to post call log artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_messages(self, skype_db, helper): + #Query for messages and iterate row by row adding + #each message artifact + try: + messages_parser = SkypeMessagesParser(skype_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + #Error parsing Skype db + self._logger.log(Level.WARNING, + "Error parsing Skype database for message artifacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Severe error trying to add to case database.. case is not complete. + #These exceptions are thrown by the CommunicationArtifactsHelper. + self._logger.log(Level.SEVERE, + "Failed to add message artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Failed to post notification to blackboard + self._logger.log(Level.WARNING, + "Failed to post message artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + class SkypeCallLogsParser(TskCallLogsParser): """ Extracts TSK_CALLLOG information from the Skype database. From 97a2d081238227a775f37546cfcbc12f48f83a96 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 19 Sep 2019 16:21:00 -0400 Subject: [PATCH 40/55] Address review comments. --- InternalPythonModules/android/imo.py | 38 ++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 5a653ff9d5..36b90bc3be 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -56,10 +56,14 @@ and adds artifacts to the case. class IMOAnalyzer(general.AndroidComponentAnalyzer): def __init__(self): self._logger = Logger.getLogger(self.__class__.__name__) + self._PACKAGE_NAME = "com.imo.android.imous" + self._PARSER_NAME = "IMO Parser" + self._MESSAGE_TYPE = "IMO Message" + self._VERSION = "9.8.0" def analyze(self, dataSource, fileManager, context): selfAccountAddress = None - accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", True, "com.imo.android.imous") + accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", True, self._PACKAGE_NAME) for accountDb in accountDbs: try: accountResultSet = accountDb.runQuery("SELECT uid, name FROM account") @@ -71,16 +75,26 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): selfAccountAddress = Account.Address(accountResultSet.getString("uid"), accountResultSet.getString("name")) except SQLException as ex: - self._logger.log(Level.SEVERE, "Error processing query result for account", ex) + self._logger.log(Level.WARNING, "Error processing query result for account", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) finally: accountDb.close() - friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", True, "com.imo.android.imous") + friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", True, self._PACKAGE_NAME) for friendsDb in friendsDbs: try: - friendsDBHelper = CommunicationArtifactsHelper(Case.getCurrentCase().getSleuthkitCase(), - "IMO Parser", friendsDb.getDBFile(), + current_case = Case.getCurrentCaseThrows() + if selfAccountAddress is not None: + friendsDBHelper = CommunicationArtifactsHelper(current_case.getSleuthkitCase(), + self._PARSER_NAME, + friendsDb.getDBFile(), Account.Type.IMO, Account.Type.IMO, selfAccountAddress ) + else: + friendsDBHelper = CommunicationArtifactsHelper(current_case.getSleuthkitCase(), + self._PARSER_NAME, + friendsDb.getDBFile(), + Account.Type.IMO + ) contactsResultSet = friendsDb.runQuery("SELECT buid, name FROM friends") if contactsResultSet is not None: while contactsResultSet.next(): @@ -121,7 +135,7 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): messageArtifact = friendsDBHelper.addMessage( - "IMO Message", + self._MESSAGE_TYPE, direction, fromAddress, toAddress, @@ -137,8 +151,16 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): except SQLException as ex: self._logger.log(Level.WARNING, "Error processing query result for IMO friends", ex) - except (TskCoreException, BlackboardException) as ex: - self._logger.log(Level.WARNING, "Failed to message artifacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, "Failed to add IMO message artifacts.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, "Failed to post artifacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) finally: friendsDb.close() From edd7fb99fd111093d5a9567f3feae448810b93b0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 17:16:59 -0400 Subject: [PATCH 41/55] Updated line and the parser templates with the latest infra changes. Also implemented bug fixes for line --- .../android/TskCallLogsParser.py | 17 +- .../android/TskMessagesParser.py | 17 +- InternalPythonModules/android/line.py | 262 +++++++++++------- 3 files changed, 182 insertions(+), 114 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 8c61070693..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,15 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - self._DEFAULT_LONG = -1 + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index 1dc9871329..f58448943f 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -16,7 +16,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - from java.io import File from java.lang import Class from java.lang import ClassNotFoundException @@ -31,8 +30,8 @@ from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil -from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB + from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile @@ -40,11 +39,15 @@ from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from TskContactsParser import TskContactsParser from TskMessagesParser import TskMessagesParser from TskCallLogsParser import TskCallLogsParser -from general import appendAttachmentList import traceback import general @@ -63,67 +66,121 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): try: contact_and_message_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "naver_line", True, self._LINE_PACKAGE_NAME) + "naver_line", True, self._LINE_PACKAGE_NAME) calllog_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "call_history", True, self._LINE_PACKAGE_NAME) + "call_history", True, self._LINE_PACKAGE_NAME) for contact_and_message_db in contact_and_message_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) - contacts_parser = LineContactsParser(contact_and_message_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() - - messages_parser = LineMessagesParser(contact_and_message_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() + self.parse_contacts(contact_and_message_db, helper) + self.parse_messages(contact_and_message_db, helper) contact_and_message_db.close() for calllog_db in calllog_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, - calllog_db.getDBFile(), Account.Type.LINE) - calllog_db.attachDatabase(dataSource, - "naver_line", calllog_db.getDBFile().getParentPath(), "naver") + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + calllog_db.getDBFile(), Account.Type.LINE) - calllog_parser = LineCallLogsParser(calllog_db) - while calllog_parser.next(): - helper.addCalllog( - calllog_parser.get_call_direction(), - calllog_parser.get_phone_number_from(), - calllog_parser.get_phone_number_to(), - calllog_parser.get_call_start_date_time(), - calllog_parser.get_call_end_date_time(), - calllog_parser.get_call_type() - ) - calllog_db.detachDatabase("naver") - calllog_parser.close() + calllog_db.attachDatabase( + dataSource, "naver_line", + calllog_db.getDBFile().getParentPath(), "naver") + self.parse_calllogs(calllog_db, helper) calllog_db.close() - except (SQLException, TskCoreException) as ex: + except NoCurrentCaseException as ex: # Error parsing Line databases. self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) self._logger.log(Level.WARNING, traceback.format_exc()) + def parse_contacts(self, contacts_db, helper): + try: + contacts_parser = LineContactsParser(contacts_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing the Line App Database for contacts", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifact to case database... case is not complete. + self._logger.log(Level.SEVERE, + "Error adding Line contact artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard + self._logger.log(Level.WARNING, + "Error posting Line contact artifacts to blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_calllogs(self, calllogs_db, helper): + try: + calllog_parser = LineCallLogsParser(calllogs_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing the Line App Database for calllogs", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifact to case database... case is not complete. + self._logger.log(Level.SEVERE, + "Error adding Line calllog artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard + self._logger.log(Level.WARNING, + "Error posting Line calllog artifacts to blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_messages(self, messages_db, helper): + try: + messages_parser = LineMessagesParser(messages_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing the Line App Database for messages.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifact to case database... case is not complete. + self._logger.log(Level.SEVERE, + "Error adding Line message artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard + self._logger.log(Level.WARNING, + "Error posting Line message artifacts to blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + class LineCallLogsParser(TskCallLogsParser): """ Parses out TSK_CALLLOG information from the Line database. @@ -148,7 +205,6 @@ class LineCallLogsParser(TskCallLogsParser): ) self._OUTGOING_CALL_TYPE = "O" self._INCOMING_CALL_TYPE = "I" - self._had_error = False self._VIDEO_CALL_TYPE = "V" self._AUDIO_CALL_TYPE = "A" @@ -162,14 +218,12 @@ class LineCallLogsParser(TskCallLogsParser): try: return long(self.result_set.getString("start_time")) / 1000 except ValueError as ve: - self._had_error = True return super(LineCallLogsParser, self).get_call_start_date_time() def get_call_end_date_time(self): try: return long(self.result_set.getString("end_time")) / 1000 except ValueError as ve: - self._had_error = True return super(LineCallLogsParser, self).get_call_end_date_time() def get_phone_number_to(self): @@ -191,9 +245,6 @@ class LineCallLogsParser(TskCallLogsParser): return self.AUDIO_CALL return super(LineCallLogsParser, self).get_call_type() - def has_incomplete_results(self): - return self._had_error - class LineContactsParser(TskContactsParser): """ Parses out TSK_CONTACT information from the Line database. @@ -225,42 +276,52 @@ class LineMessagesParser(TskMessagesParser): def __init__(self, message_db): super(LineMessagesParser, self).__init__(message_db.runQuery( - """ - SELECT all_contacts.name, - all_contacts.id, - all_contacts.members, - CH.from_mid, - CH.content, - CH.created_time, - CH.attachement_type, - CH.attachement_local_uri, - CH.status - FROM (SELECT G.name, - group_members.id, - group_members.members - FROM (SELECT id, - group_concat(m_id) AS members - FROM membership - GROUP BY id) AS group_members - JOIN groups AS G - ON G.id = group_members.id - UNION - SELECT server_name, - m_id, - NULL - FROM contacts) AS all_contacts - JOIN chat_history AS CH - ON CH.chat_id = all_contacts.id - WHERE attachement_type != 6 - """ - ) + """ + SELECT contact_list_with_groups.name, + contact_list_with_groups.id, + contact_list_with_groups.members, + contact_list_with_groups.member_names, + CH.from_mid, + C.server_name AS from_name, + CH.content, + CH.created_time, + CH.attachement_type, + CH.attachement_local_uri, + CH.status + FROM (SELECT G.name, + group_members.id, + group_members.members, + group_members.member_names + FROM (SELECT id, + group_concat(M.m_id) AS members, + group_concat(replace(C.server_name, + ",", + "")) as member_names + FROM membership AS M + JOIN contacts as C + ON M.m_id = C.m_id + GROUP BY id) AS group_members + JOIN groups AS G + ON G.id = group_members.id + UNION + SELECT server_name, + m_id, + NULL, + NULL + FROM contacts) AS contact_list_with_groups + JOIN chat_history AS CH + ON CH.chat_id = contact_list_with_groups.id + LEFT JOIN contacts as C + ON C.m_id = CH.from_mid + WHERE attachement_type != 6 + """ + ) ) self._LINE_MESSAGE_TYPE = "Line Message" #From the limited test data, it appeared that incoming #was only associated with a 1 status. Status # 3 and 7 #was only associated with outgoing. self._INCOMING_MESSAGE_TYPE = 1 - self._had_error = False def get_message_type(self): return self._LINE_MESSAGE_TYPE @@ -271,16 +332,15 @@ class LineMessagesParser(TskMessagesParser): #Get time in seconds (created_time is stored in ms from epoch) return long(created_time) / 1000 except ValueError as ve: - self._had_error = True return super(LineMessagesParser, self).get_message_date_time() def get_message_text(self): content = self.result_set.getString("content") attachment_uri = self.result_set.getString("attachement_local_uri") if attachment_uri is not None and content is not None: - return appendAttachmentList(content, [attachment_uri]) + return general.appendAttachmentList(content, [attachment_uri]) elif attachment_uri is not None and content is None: - return appendAttachmentList("", [attachment_uri]) + return general.appendAttachmentList("", [attachment_uri]) return content def get_message_direction(self): @@ -290,21 +350,27 @@ class LineMessagesParser(TskMessagesParser): def get_phone_number_from(self): if self.get_message_direction() == self.INCOMING: - group = self.result_set.getString("members") - if group is None: - return Account.Address(self.result_set.getString("from_mid"), - self.result_set.getString("name")) return Account.Address(self.result_set.getString("from_mid"), - self.result_set.getString("name")) + self.result_set.getString("from_name")) return super(LineMessagesParser, self).get_phone_number_from() def get_phone_number_to(self): if self.get_message_direction() == self.OUTGOING: group = self.result_set.getString("members") - if group is None: - return Account.Address(self.result_set.getString("id"), - self.result_set.getString("name")) - return Account.Address(group, self.result_set.getString("name")) + if group is not None: + group = group.split(",") + names = self.result_set.getString("member_names").split(",") + + recipients = [] + + for recipient_id, recipient_name in zip(group, names): + recipients.append(Account.Address(recipient_id, recipient_name)) + + return recipients + + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("name")) + return super(LineMessagesParser, self).get_phone_number_to() def get_thread_id(self): From a71fb3b283d6a7168f081953baa8a695c2c0b725 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 17:24:37 -0400 Subject: [PATCH 42/55] Changed where the bail out happens for nocurrentcase.. also closed all databases after parsing rather than during --- InternalPythonModules/android/textnow.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index 1471f720a5..5ca6d4e909 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -83,9 +83,9 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, "textnow_data.db", True, self._TEXTNOW_PACKAGE_NAME) - - for textnow_db in textnow_dbs: - try: + + try: + for textnow_db in textnow_dbs: current_case = Case.getCurrentCaseThrows() helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, @@ -94,11 +94,12 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): self.parse_contacts(textnow_db, helper) self.parse_calllogs(textnow_db, helper) self.parse_messages(textnow_db, helper) - except NoCurrentCaseException as ex: - self._logger.log(Level.WARNING, "No case currently open.", ex) - self._logger.log(Level.WARNING, traceback.format_exc()) - finally: - textnow_db.close() + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + for textnow_db in textnow_dbs: + textnow_db.close() def parse_contacts(self, textnow_db, helper): #Query for contacts and iterate row by row adding From 596a12531c1617dcda65dfc23d6b98a8dc99d6c0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 17:31:24 -0400 Subject: [PATCH 43/55] Changed where nocurrentcase bails out, closed all dbs after processing rather than during --- InternalPythonModules/android/skype.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index 23c8114c87..e93782e4a7 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -110,9 +110,8 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): #package. skype_dbs = AppSQLiteDB.findAppDatabases(dataSource, "live:", False, self._SKYPE_PACKAGE_NAME) - - for skype_db in skype_dbs: - try: + try: + for skype_db in skype_dbs: #Attempt to get the user account id from the database user_account_instance = None try: @@ -138,11 +137,12 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer): self.parse_contacts(skype_db, helper) self.parse_calllogs(skype_db, helper) self.parse_messages(skype_db, helper) - except NoCurrentCaseException as ex: - self._logger.log(Level.WARNING, "No case currently open.", ex) - self._logger.log(Level.WARNING, traceback.format_exc()) - finally: - skype_db.close() + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + for skype_db in skype_dbs: + skype_db.close() def parse_contacts(self, skype_db, helper): #Query for contacts and iterate row by row adding From d6c07d5055caf99746b89eb0ec3609fbdef51c39 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 17:48:23 -0400 Subject: [PATCH 44/55] Bug fixes --- InternalPythonModules/android/textnow.py | 82 +++++++++++++----------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index 5ca6d4e909..216864f2e1 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -296,46 +296,47 @@ class TextNowMessagesParser(TskMessagesParser): """ super(TextNowMessagesParser, self).__init__(message_db.runQuery( """ - SELECT CASE - WHEN message_direction == 2 THEN "" - WHEN to_addresses IS NULL THEN M.contact_value - ELSE contact_name - end from_address, - CASE - WHEN message_direction == 1 THEN "" - WHEN to_addresses IS NULL THEN M.contact_value - ELSE to_addresses - end to_address, - message_direction, - message_text, - M.READ, - M.date, - M.attach, - thread_id - FROM (SELECT group_info.contact_value, - group_info.to_addresses, - G._id AS thread_id - FROM (SELECT GM.contact_value, - Group_concat(GM.member_contact_value) AS to_addresses - FROM group_members AS GM - GROUP BY GM.contact_value) AS group_info - JOIN groups AS G - ON G.contact_value = group_info.contact_value - UNION - SELECT c.contact_value, - NULL, - -1 - FROM contacts AS c) AS to_from_map - JOIN messages AS M - ON M.contact_value = to_from_map.contact_value - WHERE message_type NOT IN ( 102, 100 ) + + SELECT CASE + WHEN message_direction == 2 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE contact_name + end from_address, + CASE + WHEN message_direction == 1 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE to_addresses + end to_address, + message_direction, + message_text, + M.READ, + M.date, + M.attach, + thread_id + FROM (SELECT group_info.contact_value, + group_info.to_addresses, + G.contact_value AS thread_id + FROM (SELECT GM.contact_value, + Group_concat(GM.member_contact_value) AS to_addresses + FROM group_members AS GM + GROUP BY GM.contact_value) AS group_info + JOIN groups AS G + ON G.contact_value = group_info.contact_value + UNION + SELECT c.contact_value, + NULL, + "-1" + FROM contacts AS c) AS to_from_map + JOIN messages AS M + ON M.contact_value = to_from_map.contact_value + WHERE message_type NOT IN ( 102, 100 ) """ ) ) self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" self._INCOMING_MESSAGE_TYPE = 1 self._OUTGOING_MESSAGE_TYPE = 2 - self._UNKNOWN_THREAD_ID = -1 + self._UNKNOWN_THREAD_ID = "-1" def get_message_type(self): return self._TEXTNOW_MESSAGE_TYPE @@ -355,8 +356,13 @@ class TextNowMessagesParser(TskMessagesParser): def get_phone_number_to(self): if self.result_set.getString("to_address") == "": return super(TextNowMessagesParser, self).get_phone_number_to() - return Account.Address(self.result_set.getString("to_address"), - self.result_set.getString("to_address")) + recipients = self.result_set.getString("to_address").split(",") + + recipient_accounts = [] + for recipient in recipients: + recipient_accounts.append(Account.Address(recipient, recipient)) + + return recipient_accounts def get_message_date_time(self): #convert ms to s @@ -380,7 +386,7 @@ class TextNowMessagesParser(TskMessagesParser): return text def get_thread_id(self): - thread_id = self.result_set.getInt("thread_id") + thread_id = self.result_set.getString("thread_id") if thread_id == self._UNKNOWN_THREAD_ID: return super(TextNowMessagesParser, self).get_thread_id() - return str(thread_id) + return thread_id From 2bb38065a69395adb7a3f626d27a1acea92610b3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 18:16:06 -0400 Subject: [PATCH 45/55] Updated viber to new infra changes and changed how exception handling is managed --- .../android/TskCallLogsParser.py | 16 +- .../android/TskMessagesParser.py | 17 +- InternalPythonModules/android/viber.py | 168 ++++++++++++------ 3 files changed, 130 insertions(+), 71 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 763ba3c15f..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,14 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index e6949b169e..5689b86be5 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -16,7 +16,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - from java.io import File from java.lang import Class from java.lang import ClassNotFoundException @@ -26,18 +25,26 @@ from java.sql import ResultSet from java.sql import SQLException from java.sql import Statement from java.util.logging import Level +from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser @@ -49,6 +56,14 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): """ Parses the Viber App databases for TSK contacts, message and calllog artifacts. + + The Viber v11.5.0 database structure is as follows: + - People can take part in N conversation(s). A conversation can have M + members and messages are exchanged in a conversation. + - Viber has a conversation table, a participant table (the people/members in the above + analogy) and a messages table. + - Each row of the participants table maps a person to a conversation_id + - Each row in the messages table has a from participant id and a conversation id. """ def __init__(self): @@ -71,61 +86,110 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): #Extract TSK_CONTACT and TSK_CALLLOG information for contact_and_calllog_db in contact_and_calllog_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, contact_and_calllog_db.getDBFile(), Account.Type.VIBER) - - contacts_parser = ViberContactsParser(contact_and_calllog_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() - - calllog_parser = ViberCallLogsParser(contact_and_calllog_db) - while calllog_parser.next(): - helper.addCalllog( - calllog_parser.get_call_direction(), - calllog_parser.get_phone_number_from(), - calllog_parser.get_phone_number_to(), - calllog_parser.get_call_start_date_time(), - calllog_parser.get_call_end_date_time(), - calllog_parser.get_call_type() - ) - calllog_parser.close() - - contact_and_calllog_db.close() + self.parse_contacts(contact_and_calllog_db, helper) + self.parse_calllogs(contact_and_calllog_db, helper) #Extract TSK_MESSAGE information for message_db in message_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, message_db.getDBFile(), Account.Type.VIBER) + self.parse_messages(message_db, helper) - messages_parser = ViberMessagesParser(message_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + + for message_db in messages_db: + message_db.close() + + for contact_and_calllog_db in contact_and_calllog_dbs: + contact_and_calllog_db.close() - message_db.close() - except (SQLException, TskCoreException) as ex: - #Error parsing Viber db - self._logger.log(Level.WARNING, "Error parsing Viber Databases", ex) + def parse_contacts(self, contacts_db, helper): + try: + contacts_parser = ViberContactsParser(contacts_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the viber database for contacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding viber contacts artifact to case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exec()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting viber contacts artifact to the blackboard.", ex) self._logger.log(Level.WARNING, traceback.format_exec()) + def parse_calllogs(self, calllogs_db, helper): + try: + calllog_parser = ViberCallLogsParser(calllogs_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the viber database for calllogs.", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding viber calllogs artifact to case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exec()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting viber calllogs artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + + + def parse_messages(self, messages_db, helper): + try: + messages_parser = ViberMessagesParser(messages_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the viber database for messages.", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding viber messages artifact to case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exec()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting viber messages artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + class ViberCallLogsParser(TskCallLogsParser): """ Extracts TSK_CALLLOG information from the Viber database. @@ -227,14 +291,6 @@ class ViberMessagesParser(TskMessagesParser): def __init__(self, message_db): """ - For our purposes, the Viber datamodel is as follows: - - People can take part in N conversation(s). A conversation can have M - members and messages are exchanged in a conversation. - - Viber has a conversation table, a participant table (the people/members in the above - analogy) and a messages table. - - Each row of the participants table maps a person to a conversation_id - - Each row in the messages table has a from participant id and a conversation id. - The query below does the following: - The first two inner joins on participants and participants_info build the 1 to many (M) mappings between the sender and the recipients for each From e3142149054852bb0170de50e0581f87d9ad81f0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 19:51:50 -0400 Subject: [PATCH 46/55] Brought the code up to date with the api, fixed bugs --- .../android/TskCallLogsParser.py | 16 +- .../android/TskMessagesParser.py | 17 +- InternalPythonModules/android/whatsapp.py | 234 +++++++++++------- 3 files changed, 167 insertions(+), 100 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 763ba3c15f..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,14 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 9ae57c09d6..0582a558c7 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -16,7 +16,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - from java.io import File from java.lang import Class from java.lang import ClassNotFoundException @@ -26,22 +25,29 @@ from java.sql import ResultSet from java.sql import SQLException from java.sql import Statement from java.util.logging import Level +from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser -from general import appendAttachmentList import traceback import general @@ -67,79 +73,128 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): try: contact_dbs = AppSQLiteDB.findAppDatabases(dataSource, "wa.db", True, self._WHATSAPP_PACKAGE_NAME) - message_dbs = AppSQLiteDB.findAppDatabases(dataSource, + calllog_and_message_dbs = AppSQLiteDB.findAppDatabases(dataSource, "msgstore.db", True, self._WHATSAPP_PACKAGE_NAME) #Extract TSK_CONTACT information for contact_db in contact_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, contact_db.getDBFile(), Account.Type.WHATSAPP) + self.parse_contacts(contact_db, helper) - contacts_parser = WhatsAppContactsParser(contact_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() + for calllog_and_message_db in calllog_and_message_dbs: + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + calllog_and_message_db.getDBFile(), Account.Type.WHATSAPP) + calllog_and_message_db.attachDatabase(dataSource, "wa.db", + calllog_and_message_db.getDBFile().getParentPath(), "wadb") + self.parse_calllogs(calllog_and_message_db, helper) + self.parse_messages(calllog_and_message_db, helper) - contact_db.close() - - for message_db in message_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, - message_db.getDBFile(), Account.Type.WHATSAPP) - - message_db.attachDatabase(dataSource, "wa.db", - message_db.getDBFile().getParentPath(), "wadb") - - messages_parser = WhatsAppMessagesParser(message_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() - - group_calllogs_parser = WhatsAppGroupCallLogsParser(message_db) - while group_calllogs_parser.next(): - helper.addCalllog( - group_calllogs_parser.get_call_direction(), - group_calllogs_parser.get_phone_number_from(), - group_calllogs_parser.get_phone_number_to(), - group_calllogs_parser.get_call_start_date_time(), - group_calllogs_parser.get_call_end_date_time(), - group_calllogs_parser.get_call_type() - ) - group_calllogs_parser.close() - - single_calllogs_parser = WhatsAppSingleCallLogsParser(message_db) - while single_calllogs_parser.next(): - helper.addCalllog( - single_calllogs_parser.get_call_direction(), - single_calllogs_parser.get_phone_number_from(), - single_calllogs_parser.get_phone_number_to(), - single_calllogs_parser.get_call_start_date_time(), - single_calllogs_parser.get_call_end_date_time(), - single_calllogs_parser.get_call_type() - ) - single_calllogs_parser.close() - - message_db.close() - except (SQLException, TskCoreException) as ex: - #Error parsing WhatsApp db - self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) + except NoCurrentCaseException as ex: + #If there is no current case, bail out immediately. + self._logger.log(Level.WARNING, "No case currently open.", ex) self._logger.log(Level.WARNING, traceback.format_exec()) + + #Clean up open file handles. + for contact_db in contact_dbs: + contact_db.close() + + for calllog_and_message_db in calllog_and_message_dbs: + calllog_and_message_db.close() + + def parse_contacts(self, contacts_db, helper): + try: + contacts_parser = WhatsAppContactsParser(contacts_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the whatsapp database for contacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding whatsapp contact artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting contact artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_calllogs(self, calllogs_db, helper): + try: + single_calllogs_parser = WhatsAppSingleCallLogsParser(calllogs_db) + while single_calllogs_parser.next(): + helper.addCalllog( + single_calllogs_parser.get_call_direction(), + single_calllogs_parser.get_phone_number_from(), + single_calllogs_parser.get_phone_number_to(), + single_calllogs_parser.get_call_start_date_time(), + single_calllogs_parser.get_call_end_date_time(), + single_calllogs_parser.get_call_type() + ) + single_calllogs_parser.close() + + group_calllogs_parser = WhatsAppGroupCallLogsParser(calllogs_db) + while group_calllogs_parser.next(): + helper.addCalllog( + group_calllogs_parser.get_call_direction(), + group_calllogs_parser.get_phone_number_from(), + group_calllogs_parser.get_phone_number_to(), + group_calllogs_parser.get_call_start_date_time(), + group_calllogs_parser.get_call_end_date_time(), + group_calllogs_parser.get_call_type() + ) + group_calllogs_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the whatsapp database for calllogs.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding whatsapp calllog artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting calllog artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_messages(self, messages_db, helper): + try: + messages_parser = WhatsAppMessagesParser(messages_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the whatsapp database for contacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding whatsapp contact artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting contact artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) class WhatsAppGroupCallLogsParser(TskCallLogsParser): """ @@ -150,17 +205,19 @@ class WhatsAppGroupCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(WhatsAppGroupCallLogsParser, self).__init__(calllog_db.runQuery( """ - SELECT CL.video_call, - CL.timestamp, - CL.duration, - CL.from_me, - J.raw_string as from_num, - group_concat(J.raw_string) AS group_members - FROM call_log_participant_v2 AS CLP - JOIN call_log AS CL - ON CL._id = CLP.call_log_row_id - JOIN jid AS J - ON J._id = CLP.jid_row_id + SELECT CL.video_call, + CL.timestamp, + CL.duration, + CL.from_me, + J1.raw_string AS from_id, + group_concat(J.raw_string) AS group_members + FROM call_log_participant_v2 AS CLP + JOIN call_log AS CL + ON CL._id = CLP.call_log_row_id + JOIN jid AS J + ON J._id = CLP.jid_row_id + JOIN jid as J1 + ON J1._id = CL.jid_row_id GROUP BY CL._id """ ) @@ -176,7 +233,7 @@ class WhatsAppGroupCallLogsParser(TskCallLogsParser): def get_phone_number_from(self): if self.get_call_direction() == self.INCOMING_CALL: - sender = self.result_set.getString("from_num") + sender = self.result_set.getString("from_id") return Account.Address(sender, sender) return super(WhatsAppGroupCallLogsParser, self).get_phone_number_from() @@ -349,12 +406,19 @@ class WhatsAppMessagesParser(TskMessagesParser): return self._WHATSAPP_MESSAGE_TYPE def get_phone_number_to(self): - group = self.result_set.getString("recipients") - if group is not None: - return Account.Address(self.result_set.getString("id"), group) if self.get_message_direction() == self.OUTGOING: + group = self.result_set.getString("recipients") + if group is not None: + group = group.split(",") + + recipients = [] + for token in group: + recipients.append(Account.Address(token, token)) + + return recipients + return Account.Address(self.result_set.getString("id"), - self.result_set.getString("id")) + self.result_set.getString("id")) return super(WhatsAppMessagesParser, self).get_phone_number_to() def get_phone_number_from(self): @@ -387,7 +451,7 @@ class WhatsAppMessagesParser(TskMessagesParser): mime_type = self.result_set.getString("attachment_mimetype") if mime_type is not None: attachment += "\nMIME type: " + mime_type - return appendAttachmentList(message, [attachment]) + return general.appendAttachmentList(message, [attachment]) return message def get_thread_id(self): From a8827a304ddb2de33a09928f70f45a8c414812df Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 19 Sep 2019 19:53:45 -0400 Subject: [PATCH 47/55] Fixed missing import. --- InternalPythonModules/android/imo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 36b90bc3be..714c029445 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -29,6 +29,7 @@ from java.util.logging import Level from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB From 4f2b0d50214441c004d33ececdf0fd1562b623a0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 19:55:28 -0400 Subject: [PATCH 48/55] Fixed type in traceback --- InternalPythonModules/android/viber.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 5689b86be5..6a7e4b2451 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -103,7 +103,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): except NoCurrentCaseException as ex: self._logger.log(Level.WARNING, "No case currently open.", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, traceback.format_exc()) for message_db in messages_db: message_db.close() @@ -126,15 +126,15 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): contacts_parser.close() except SQLException as ex: self._logger.log(Level.WARNING, "Error querying the viber database for contacts.", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, traceback.format_exc()) except TskCoreException as ex: self._logger.log(Level.SEVERE, "Error adding viber contacts artifact to case database.", ex) - self._logger.log(Level.SEVERE, traceback.format_exec()) + self._logger.log(Level.SEVERE, traceback.format_exc()) except BlackboardException as ex: self._logger.log(Level.WARNING, "Error posting viber contacts artifact to the blackboard.", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, traceback.format_exc()) def parse_calllogs(self, calllogs_db, helper): try: @@ -151,15 +151,15 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): calllog_parser.close() except SQLException as ex: self._logger.log(Level.WARNING, "Error querying the viber database for calllogs.", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, traceback.format_exc()) except TskCoreException as ex: self._logger.log(Level.SEVERE, "Error adding viber calllogs artifact to case database.", ex) - self._logger.log(Level.SEVERE, traceback.format_exec()) + self._logger.log(Level.SEVERE, traceback.format_exc()) except BlackboardException as ex: self._logger.log(Level.WARNING, "Error posting viber calllogs artifact to the blackboard.", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, traceback.format_exc()) def parse_messages(self, messages_db, helper): @@ -180,15 +180,15 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): messages_parser.close() except SQLException as ex: self._logger.log(Level.WARNING, "Error querying the viber database for messages.", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, traceback.format_exc()) except TskCoreException as ex: self._logger.log(Level.SEVERE, "Error adding viber messages artifact to case database.", ex) - self._logger.log(Level.SEVERE, traceback.format_exec()) + self._logger.log(Level.SEVERE, traceback.format_exc()) except BlackboardException as ex: self._logger.log(Level.WARNING, "Error posting viber messages artifact to the blackboard.", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, traceback.format_exc()) class ViberCallLogsParser(TskCallLogsParser): """ From 4c93fae369e83c1313dfd7d1be059754d6d4012a Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 19 Sep 2019 20:07:22 -0400 Subject: [PATCH 49/55] Fixed missing import. --- InternalPythonModules/android/shareit.py | 1 + InternalPythonModules/android/xender.py | 1 + InternalPythonModules/android/zapya.py | 1 + 3 files changed, 3 insertions(+) diff --git a/InternalPythonModules/android/shareit.py b/InternalPythonModules/android/shareit.py index bccc9b9a3b..937a663393 100644 --- a/InternalPythonModules/android/shareit.py +++ b/InternalPythonModules/android/shareit.py @@ -29,6 +29,7 @@ from java.util.logging import Level from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB diff --git a/InternalPythonModules/android/xender.py b/InternalPythonModules/android/xender.py index e3c72f33e2..cdc520fb11 100644 --- a/InternalPythonModules/android/xender.py +++ b/InternalPythonModules/android/xender.py @@ -29,6 +29,7 @@ from java.util.logging import Level from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB diff --git a/InternalPythonModules/android/zapya.py b/InternalPythonModules/android/zapya.py index 672795c076..230405075d 100644 --- a/InternalPythonModules/android/zapya.py +++ b/InternalPythonModules/android/zapya.py @@ -29,6 +29,7 @@ from java.util.logging import Level from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB From 6de1e2ab40920e7d8e1e84231b84d01ed9a73286 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:00:50 -0400 Subject: [PATCH 50/55] Fixed line bug --- InternalPythonModules/android/line.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index f58448943f..c6520edf43 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -350,8 +350,10 @@ class LineMessagesParser(TskMessagesParser): def get_phone_number_from(self): if self.get_message_direction() == self.INCOMING: - return Account.Address(self.result_set.getString("from_mid"), - self.result_set.getString("from_name")) + from_mid = self.result_set.getString("from_mid") + if from_mid is not None: + return Account.Address(from_mid, + self.result_set.getString("from_name")) return super(LineMessagesParser, self).get_phone_number_from() def get_phone_number_to(self): From 3dcab41cbd0d7e7e1a437cda88ba0f711dd39cdc Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:06:49 -0400 Subject: [PATCH 51/55] Fixed sql exception and close statements --- InternalPythonModules/android/line.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index c6520edf43..c87ef3477c 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -75,27 +75,26 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) - self.parse_contacts(contact_and_message_db, helper) self.parse_messages(contact_and_message_db, helper) - contact_and_message_db.close() for calllog_db in calllog_dbs: current_case = Case.getCurrentCaseThrows() helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) + self.parse_calllogs(dataSource, calllog_db, helper) - calllog_db.attachDatabase( - dataSource, "naver_line", - calllog_db.getDBFile().getParentPath(), "naver") - - self.parse_calllogs(calllog_db, helper) - calllog_db.close() except NoCurrentCaseException as ex: # Error parsing Line databases. self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) self._logger.log(Level.WARNING, traceback.format_exc()) + + for contact_and_message_db in contact_and_message_dbs: + contact_and_message_db.close() + + for calllog_db in calllog_dbs: + calllog_db.close() def parse_contacts(self, contacts_db, helper): try: @@ -124,8 +123,12 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): "Error posting Line contact artifacts to blackboard.", ex) self._logger.log(Level.WARNING, traceback.format_exc()) - def parse_calllogs(self, calllogs_db, helper): + def parse_calllogs(self, dataSource, calllogs_db, helper): try: + calllogs_db.attachDatabase( + dataSource, "naver_line", + calllogs_db.getDBFile().getParentPath(), "naver") + calllog_parser = LineCallLogsParser(calllogs_db) while calllog_parser.next(): helper.addCalllog( @@ -153,6 +156,7 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): def parse_messages(self, messages_db, helper): try: + messages_parser = LineMessagesParser(messages_db) while messages_parser.next(): helper.addMessage( From bf37509b1d584bd6e030be2b113418a96b153bfb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:16:39 -0400 Subject: [PATCH 52/55] Moved missed attachDatabase refactor --- InternalPythonModules/android/whatsapp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 0582a558c7..5dfa3c8f16 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -89,8 +89,6 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, calllog_and_message_db.getDBFile(), Account.Type.WHATSAPP) - calllog_and_message_db.attachDatabase(dataSource, "wa.db", - calllog_and_message_db.getDBFile().getParentPath(), "wadb") self.parse_calllogs(calllog_and_message_db, helper) self.parse_messages(calllog_and_message_db, helper) @@ -170,6 +168,9 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): def parse_messages(self, messages_db, helper): try: + messages_db.attachDatabase(dataSource, "wa.db", + messages_db.getDBFile().getParentPath(), "wadb") + messages_parser = WhatsAppMessagesParser(messages_db) while messages_parser.next(): helper.addMessage( From 17c8ed02e548240c76c5a88f204e95b06d021d15 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:20:52 -0400 Subject: [PATCH 53/55] more refactoring --- InternalPythonModules/android/whatsapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 5dfa3c8f16..eeb561923a 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -90,7 +90,7 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): current_case.getSleuthkitCase(), self._PARSER_NAME, calllog_and_message_db.getDBFile(), Account.Type.WHATSAPP) self.parse_calllogs(calllog_and_message_db, helper) - self.parse_messages(calllog_and_message_db, helper) + self.parse_messages(dataSource, calllog_and_message_db, helper) except NoCurrentCaseException as ex: #If there is no current case, bail out immediately. @@ -166,7 +166,7 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): "Error posting calllog artifact to the blackboard.", ex) self._logger.log(Level.WARNING, traceback.format_exc()) - def parse_messages(self, messages_db, helper): + def parse_messages(self, dataSource, messages_db, helper): try: messages_db.attachDatabase(dataSource, "wa.db", messages_db.getDBFile().getParentPath(), "wadb") From 5b92d1a82dcc0c0cb54c2f7a90a1bc95b0fdc3df Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:45:36 -0400 Subject: [PATCH 54/55] Attachment fix --- InternalPythonModules/android/skype.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/InternalPythonModules/android/skype.py b/InternalPythonModules/android/skype.py index e93782e4a7..f9bd6f5466 100644 --- a/InternalPythonModules/android/skype.py +++ b/InternalPythonModules/android/skype.py @@ -388,7 +388,6 @@ class SkypeMessagesParser(TskMessagesParser): contacts_list_with_groups.participants, time, content, - file_name, device_gallery_path, is_sender_me, person_id as sender_id, @@ -467,13 +466,11 @@ class SkypeMessagesParser(TskMessagesParser): content = self.result_set.getString("content") if content is not None: - file_name = self.result_set.getString("file_name") file_path = self.result_set.getString("device_gallery_path") #if a file name and file path are associated with a message, append it - if file_name is not None and file_path is not None: - attachment = "File Name: "+file_name +"\n"+ "File Path: "+file_path - return general.appendAttachmentList(content, [attachment]) + if file_path is not None: + return general.appendAttachmentList(content, [file_path]) return content From 1dbc41420bb51393d30a3a450bc3c49dcc94a162 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 11:13:13 -0400 Subject: [PATCH 55/55] Removed mime type from attachment --- InternalPythonModules/android/whatsapp.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index eeb561923a..328f371c76 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -379,8 +379,7 @@ class WhatsAppMessagesParser(TskMessagesParser): M.timestamp AS send_timestamp, M.received_timestamp, M.remote_resource AS group_sender, - M.media_url As attachment, - M.media_mime_type as attachment_mimetype + M.media_url As attachment FROM (SELECT jid, recipients FROM wadb.wa_contacts AS WC @@ -449,9 +448,6 @@ class WhatsAppMessagesParser(TskMessagesParser): message = self.result_set.getString("content") attachment = self.result_set.getString("attachment") if attachment is not None: - mime_type = self.result_set.getString("attachment_mimetype") - if mime_type is not None: - attachment += "\nMIME type: " + mime_type return general.appendAttachmentList(message, [attachment]) return message