From 19526ba5a4d037c1ff25945c717f9e095a14c880 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 12:09:00 -0400 Subject: [PATCH 1/9] Copied infrastructure into new branch --- .../android/ResultSetIterator.py | 35 ++++++++++ .../android/TskCallLogsParser.py | 58 ++++++++++++++++ .../android/TskContactsParser.py | 49 +++++++++++++ .../android/TskMessagesParser.py | 68 +++++++++++++++++++ 4 files changed, 210 insertions(+) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..66ea27eee3 --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,58 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self.INCOMING_CALL = "Incoming" + self.OUTGOING_CALL = "Outgoing" + self._DEFAULT_STRING = "" + + def get_account_name(self): + return self._DEFAULT_STRING + + def get_call_direction(self): + return self._DEFAULT_STRING + + def get_phone_number_from(self): + return self._DEFAULT_STRING + + def get_phone_number_to(self): + return self._DEFAULT_STRING + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_contact_name(self): + return self._DEFAULT_STRING diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..e3edbb25c8 --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,68 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self.INCOMING_MSG = "Incoming" + self.OUTGOING_MSG = "Outgoing" + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_TEXT + + def get_phone_number_from(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_MSG_READ_STATUS + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT From 7026b84b7aacfe973b81fffe5b1b9bb186a550eb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 12:13:31 -0400 Subject: [PATCH 2/9] Moved old whatsapp work into this branch and modified module.py to run --- InternalPythonModules/android/module.py | 3 +- InternalPythonModules/android/whatsapp.py | 203 ++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 InternalPythonModules/android/whatsapp.py diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..322700481c 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import whatsapp class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), whatsapp.WhatsAppAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py new file mode 100644 index 0000000000..cceff4494f --- /dev/null +++ b/InternalPythonModules/android/whatsapp.py @@ -0,0 +1,203 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskMessagesParser import TskMessagesParser +from TskContactsParser import TskContactsParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the WhatsApp databases for TSK contact and message artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._WHATSAPP_PACKAGE_NAME = "com.whatsapp" + self._PARSER_NAME = "WhatsApp Parser" + + def analyze(self, dataSource, fileManager, context): + """ + Extract, Transform and Load all messages and contacts from the WhatsApp databases. + """ + + try: + contact_dbs = SQLiteUtil.findAppDatabases(dataSource, "wa.db", self._WHATSAPP_PACKAGE_NAME) + message_dbs = SQLiteUtil.findAppDatabases(dataSource, "msgstore.db", self._WHATSAPP_PACKAGE_NAME) + + #Extract TSK_CONTACT information + for contact_db in contact_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_db.getDBFile(), Account.Type.WHATSAPP) + contacts_parser = WhatsAppContactsParser(contact_db) + while contacts_parser.next(): + blackboard_util.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + + for message_db in message_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, message_db.getDBFile(), Account.Type.WHATSAPP) + """ + message_db.attachDatabase(message_db.getDBFile().getParentPath(), "wa.db", "wadb") + messages_parser = WhatsAppMessagesParser(message_db) + while messages_parser.next(): + blackboard_util.addMessage( + messages_parser.get_account_id(), + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + """ + except (SQLException, TskCoreException) as ex: + #Error parsing WhatsApp db + self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + +class WhatsAppContactsParser(TskContactsParser): + """ + Extracts TSK_CONTACT information from the WhatsApp database. + TSK_CONTACT fields that are not in the WhatsApp database are given a default value + inherited from the super class. + """ + + def __init__(self, contact_db): + super(WhatsAppContactsParser, self).__init__(contact_db.runQuery( + """ + SELECT number, + CASE + WHEN given_name is NULL THEN family_name + WHEN family_name is NULL THEN given_name + ELSE given_name + || " " + || family_name + END name + FROM wa_contacts + WHERE given_name is not NULL OR family_name IS NOT NULL + """ + ) + ) + + def get_account_name(self): + return self.result_set.getString("name") + + def get_contact_name(self): + return self.result_set.getString("name") + + def get_phone(self): + return self.result_set.getString("number") + +class WhatsAppMessagesParser(TskMessagesParser): + """ + Extract TSK_MESSAGE information from the WhatsApp database. + TSK_CONTACT fields that are not in the WhatsApp database are given a default value + inherited from the super class. + """ + + def __init__(self, message_db): + super(WhatsAppMessageParser, self).__init__(message_db.runQuery( + """ + SELECT M.data AS content, + WDB.number AS number, + CASE + WHEN WDB.given_name IS NULL THEN WDB.family_name + WHEN WDB.family_name IS NULL THEN WDB.given_name + ELSE WDB.given_name + || " " + || WDB.family_name + END name, + M.key_from_me AS direction, + M.received_timestamp AS recieved_datetime, + M.timestamp AS send_datetime + FROM messages AS M + JOIN wadb.wa_contacts AS WDB + ON M.key_remote_jid = WDB.jid + """ + ) + ) + self._WHATSAPP_MESSAGE_TYPE = "WhatsApp Message" + self._INCOMING_MESSAGE_TYPE = 0 + self._OUTGOING_MESSAGE_TYPE = 1 + self._INCOMING_MSG_STRING = "Incoming" + self._OUTGOING_MSG_STRING = "Outgoing" + + def get_account_id(self): + return self.result_set.getString("name") + + def get_message_type(self): + return self._WHATSAPP_MESSAGE_TYPE + + def get_phone_number_to(self): + if self.get_message_direction() == self._OUTGOING_MSG_STRING: + return self.result_set.getString("number") + return super(WhatsAppMessageParser, self).get_phone_number_to() + + def get_phone_number_from(self): + if self.get_message_direction() == self._INCOMING_MSG_STRING: + return self.result_set.getString("number") + return super(WhatsAppMessageParser, self).get_phone_number_from() + + def get_message_direction(self): + direction = self.result_set.getInt("direction") + if direction == self._INCOMING_MESSAGE_TYPE: + return self._INCOMING_MSG_STRING + return self._OUTGOING_MSG_STRING + + def get_message_date_time(self): + #transform from ms to seconds + if get_message_direction() == self._OUTGOING_MSG_STRING: + return self.result_set.getLong("send_datetime") / 1000 + return self.result_set.getLong("received_datetime") / 1000 + + def get_message_text(self): + return self.result_set.getString("content") From a131ce17602dba62958aa8186023ada08514e12d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 15:08:43 -0400 Subject: [PATCH 3/9] Rest of the whats app implementation and some refactoring --- InternalPythonModules/android/whatsapp.py | 126 ++++++++++++---------- 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index cceff4494f..cc46bc0380 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -29,8 +29,8 @@ from java.util.logging import Level from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger -from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact @@ -57,19 +57,24 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): """ - Extract, Transform and Load all messages and contacts from the WhatsApp databases. + Extract, Transform and Load all TSK contact and message + artifacts from the WhatsApp databases. """ try: - contact_dbs = SQLiteUtil.findAppDatabases(dataSource, "wa.db", self._WHATSAPP_PACKAGE_NAME) - message_dbs = SQLiteUtil.findAppDatabases(dataSource, "msgstore.db", self._WHATSAPP_PACKAGE_NAME) + contact_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "wa.db", True, self._WHATSAPP_PACKAGE_NAME) + message_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "msgstore.db", True, self._WHATSAPP_PACKAGE_NAME) #Extract TSK_CONTACT information for contact_db in contact_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_db.getDBFile(), Account.Type.WHATSAPP) + helper = AppDBParserHelper(self._PARSER_NAME, + contact_db.getDBFile(), Account.Type.WHATSAPP) + contacts_parser = WhatsAppContactsParser(contact_db) while contacts_parser.next(): - blackboard_util.addContact( + helper.addContact( contacts_parser.get_account_name(), contacts_parser.get_contact_name(), contacts_parser.get_phone(), @@ -79,14 +84,18 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): ) contacts_parser.close() + contact_db.close() + for message_db in message_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, message_db.getDBFile(), Account.Type.WHATSAPP) - """ - message_db.attachDatabase(message_db.getDBFile().getParentPath(), "wa.db", "wadb") + helper = AppDBParserHelper(self._PARSER_NAME, + message_db.getDBFile(), Account.Type.WHATSAPP) + + message_db.attachDatabase(dataSource, "wa.db", + message_db.getDBFile().getParentPath(), "wadb") + messages_parser = WhatsAppMessagesParser(message_db) while messages_parser.next(): - blackboard_util.addMessage( - messages_parser.get_account_id(), + helper.addMessage( messages_parser.get_message_type(), messages_parser.get_message_direction(), messages_parser.get_phone_number_from(), @@ -98,7 +107,8 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - """ + + message_db.close() except (SQLException, TskCoreException) as ex: #Error parsing WhatsApp db self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) @@ -107,29 +117,21 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): class WhatsAppContactsParser(TskContactsParser): """ Extracts TSK_CONTACT information from the WhatsApp database. - TSK_CONTACT fields that are not in the WhatsApp database are given a default value - inherited from the super class. + TSK_CONTACT fields that are not in the WhatsApp database are given + a default value inherited from the super class. """ def __init__(self, contact_db): super(WhatsAppContactsParser, self).__init__(contact_db.runQuery( """ - SELECT number, - CASE - WHEN given_name is NULL THEN family_name - WHEN family_name is NULL THEN given_name - ELSE given_name - || " " - || family_name - END name - FROM wa_contacts - WHERE given_name is not NULL OR family_name IS NOT NULL + SELECT """ + _get_contacts_formatting() + """ + FROM wa_contacts AS WC """ ) ) def get_account_name(self): - return self.result_set.getString("name") + return self.get_phone() def get_contact_name(self): return self.result_set.getString("name") @@ -140,64 +142,80 @@ class WhatsAppContactsParser(TskContactsParser): class WhatsAppMessagesParser(TskMessagesParser): """ Extract TSK_MESSAGE information from the WhatsApp database. - TSK_CONTACT fields that are not in the WhatsApp database are given a default value - inherited from the super class. + TSK_CONTACT fields that are not in the WhatsApp database are given + a default value inherited from the super class. """ def __init__(self, message_db): - super(WhatsAppMessageParser, self).__init__(message_db.runQuery( + super(WhatsAppMessagesParser, self).__init__(message_db.runQuery( """ SELECT M.data AS content, - WDB.number AS number, - CASE - WHEN WDB.given_name IS NULL THEN WDB.family_name - WHEN WDB.family_name IS NULL THEN WDB.given_name - ELSE WDB.given_name - || " " - || WDB.family_name - END name, + """+_get_contacts_formatting()+""", M.key_from_me AS direction, - M.received_timestamp AS recieved_datetime, + M.received_timestamp AS received_datetime, M.timestamp AS send_datetime FROM messages AS M - JOIN wadb.wa_contacts AS WDB - ON M.key_remote_jid = WDB.jid + JOIN wadb.wa_contacts AS WC + ON M.key_remote_jid = WC.jid """ ) ) self._WHATSAPP_MESSAGE_TYPE = "WhatsApp Message" self._INCOMING_MESSAGE_TYPE = 0 self._OUTGOING_MESSAGE_TYPE = 1 - self._INCOMING_MSG_STRING = "Incoming" - self._OUTGOING_MSG_STRING = "Outgoing" - - def get_account_id(self): - return self.result_set.getString("name") def get_message_type(self): return self._WHATSAPP_MESSAGE_TYPE def get_phone_number_to(self): - if self.get_message_direction() == self._OUTGOING_MSG_STRING: - return self.result_set.getString("number") - return super(WhatsAppMessageParser, self).get_phone_number_to() + if self.get_message_direction() == self.OUTGOING_MSG: + return Account.Address(self.result_set.getString("number"), + self.result_set.getString("number")) + return super(WhatsAppMessagesParser, self).get_phone_number_to() def get_phone_number_from(self): - if self.get_message_direction() == self._INCOMING_MSG_STRING: - return self.result_set.getString("number") - return super(WhatsAppMessageParser, self).get_phone_number_from() + if self.get_message_direction() == self.INCOMING_MSG: + return Account.Address(self.result_set.getString("number"), + self.result_set.getString("number")) + return super(WhatsAppMessagesParser, self).get_phone_number_from() def get_message_direction(self): direction = self.result_set.getInt("direction") if direction == self._INCOMING_MESSAGE_TYPE: - return self._INCOMING_MSG_STRING - return self._OUTGOING_MSG_STRING + return self.INCOMING_MSG + return self.OUTGOING_MSG def get_message_date_time(self): #transform from ms to seconds - if get_message_direction() == self._OUTGOING_MSG_STRING: + if self.get_message_direction() == self.OUTGOING_MSG: return self.result_set.getLong("send_datetime") / 1000 return self.result_set.getLong("received_datetime") / 1000 def get_message_text(self): return self.result_set.getString("content") + +def _get_contacts_formatting(): + """ + This function is here to explicitly stress the point that the + formatting routine used in the contacts and messages parsers + should never differ. These fields are used to correlate in Autopsy. + + The SQL statement assumes wa_contacts table is named WC. + """ + + return """ + CASE + WHEN WC.number IS NULL THEN WC.jid + WHEN WC.number == "" THEN WC.jid + ELSE WC.number + END number, + CASE + WHEN WC.given_name IS NULL + AND WC.family_name IS NULL THEN WC.jid + WHEN WC.given_name IS NULL THEN WC.family_name + WHEN WC.family_name IS NULL THEN WC.given_name + ELSE WC.given_name + || " " + || WC.family_name + END name + """ From 8fd7abdf559e9e965b1f2e827b25e618e627da6b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sat, 14 Sep 2019 11:03:49 -0400 Subject: [PATCH 4/9] Updated infra changes --- .../android/TskCallLogsParser.py | 25 +++++++++++-------- .../android/TskMessagesParser.py | 12 ++++++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 66ea27eee3..763ba3c15f 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): """ @@ -32,27 +34,30 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) - self.INCOMING_CALL = "Incoming" - self.OUTGOING_CALL = "Outgoing" self._DEFAULT_STRING = "" + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = None + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - def get_account_name(self): - return self._DEFAULT_STRING + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO def get_call_direction(self): - return self._DEFAULT_STRING + return self._DEFAULT_DIRECTION def get_phone_number_from(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_call_start_date_time(self): return self._DEFAULT_LONG def get_call_end_date_time(self): return self._DEFAULT_LONG - - def get_contact_name(self): - return self._DEFAULT_STRING + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index e3edbb25c8..15c4166db7 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -33,18 +33,22 @@ class TskMessagesParser(ResultSetIterator): def __init__(self, result_set): super(TskMessagesParser, self).__init__(result_set) - self.INCOMING_MSG = "Incoming" - self.OUTGOING_MSG = "Outgoing" self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN - self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_ACCOUNT_ADDRESS = None + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD def get_message_type(self): return self._DEFAULT_TEXT def get_message_direction(self): - return self._DEFAULT_TEXT + return self._DEFAULT_COMMUNICATION_DIRECTION def get_phone_number_from(self): return self._DEFAULT_ACCOUNT_ADDRESS From 611a03fa09e1e0af763e5fbe73f81eb6e923cb0b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sat, 14 Sep 2019 11:04:52 -0400 Subject: [PATCH 5/9] Added version number --- InternalPythonModules/android/whatsapp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index cc46bc0380..7afdf184c2 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -54,6 +54,7 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) self._WHATSAPP_PACKAGE_NAME = "com.whatsapp" self._PARSER_NAME = "WhatsApp Parser" + self._VERSION = "2.19.244" def analyze(self, dataSource, fileManager, context): """ From 7f2464a2e59d6c7dd83b9aa6e5147587650bb448 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 12:21:05 -0400 Subject: [PATCH 6/9] Fully implemented the whatsapp parser --- InternalPythonModules/android/general.py | 11 + InternalPythonModules/android/whatsapp.py | 279 ++++++++++++++++++---- 2 files changed, 238 insertions(+), 52 deletions(-) diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 7afdf184c2..9ae57c09d6 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -41,13 +41,15 @@ from org.sleuthkit.datamodel import Account from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser +from general import appendAttachmentList import traceback import general class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): """ - Parses the WhatsApp databases for TSK contact and message artifacts. + Parses the WhatsApp databases for TSK contact, message + and calllog artifacts. """ def __init__(self): @@ -58,8 +60,8 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): """ - Extract, Transform and Load all TSK contact and message - artifacts from the WhatsApp databases. + Extract, Transform and Load all TSK contact, message + and calllog artifacts from the WhatsApp databases. """ try: @@ -109,12 +111,154 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): ) messages_parser.close() + group_calllogs_parser = WhatsAppGroupCallLogsParser(message_db) + while group_calllogs_parser.next(): + helper.addCalllog( + group_calllogs_parser.get_call_direction(), + group_calllogs_parser.get_phone_number_from(), + group_calllogs_parser.get_phone_number_to(), + group_calllogs_parser.get_call_start_date_time(), + group_calllogs_parser.get_call_end_date_time(), + group_calllogs_parser.get_call_type() + ) + group_calllogs_parser.close() + + single_calllogs_parser = WhatsAppSingleCallLogsParser(message_db) + while single_calllogs_parser.next(): + helper.addCalllog( + single_calllogs_parser.get_call_direction(), + single_calllogs_parser.get_phone_number_from(), + single_calllogs_parser.get_phone_number_to(), + single_calllogs_parser.get_call_start_date_time(), + single_calllogs_parser.get_call_end_date_time(), + single_calllogs_parser.get_call_type() + ) + single_calllogs_parser.close() + message_db.close() except (SQLException, TskCoreException) as ex: #Error parsing WhatsApp db self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) self._logger.log(Level.WARNING, traceback.format_exec()) +class WhatsAppGroupCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from group call logs + in the WhatsApp database. + """ + + def __init__(self, calllog_db): + super(WhatsAppGroupCallLogsParser, self).__init__(calllog_db.runQuery( + """ + SELECT CL.video_call, + CL.timestamp, + CL.duration, + CL.from_me, + J.raw_string as from_num, + group_concat(J.raw_string) AS group_members + FROM call_log_participant_v2 AS CLP + JOIN call_log AS CL + ON CL._id = CLP.call_log_row_id + JOIN jid AS J + ON J._id = CLP.jid_row_id + GROUP BY CL._id + """ + ) + ) + self._INCOMING_CALL_TYPE = 0 + self._OUTGOING_CALL_TYPE = 1 + self._VIDEO_CALL_TYPE = 1 + + def get_call_direction(self): + if self.result_set.getInt("from_me") == self._INCOMING_CALL_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL + + def get_phone_number_from(self): + if self.get_call_direction() == self.INCOMING_CALL: + sender = self.result_set.getString("from_num") + return Account.Address(sender, sender) + return super(WhatsAppGroupCallLogsParser, self).get_phone_number_from() + + def get_phone_number_to(self): + if self.get_call_direction() == self.OUTGOING_CALL: + group = self.result_set.getString("group_members") + members = [] + for token in group.split(","): + members.append(Account.Address(token, token)) + return members + return super(WhatsAppGroupCallLogsParser, self).get_phone_number_to() + + def get_call_start_date_time(self): + return self.result_set.getLong("timestamp") / 1000 + + def get_call_end_date_time(self): + start = self.get_call_start_date_time() + duration = self.result_set.getInt("duration") + return start + duration + + def get_call_type(self): + if self.result_set.getInt("video_call") == self._VIDEO_CALL_TYPE: + return self.VIDEO_CALL + return self.AUDIO_CALL + +class WhatsAppSingleCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from 1 to 1 call logs + in the WhatsApp database. + """ + + def __init__(self, calllog_db): + super(WhatsAppSingleCallLogsParser, self).__init__(calllog_db.runQuery( + """ + SELECT CL.timestamp, + CL.video_call, + CL.duration, + J.raw_string AS num, + CL.from_me + FROM call_log AS CL + JOIN jid AS J + ON J._id = CL.jid_row_id + WHERE CL._id NOT IN (SELECT DISTINCT call_log_row_id + FROM call_log_participant_v2) + """ + ) + ) + self._INCOMING_CALL_TYPE = 0 + self._OUTGOING_CALL_TYPE = 1 + self._VIDEO_CALL_TYPE = 1 + + def get_call_direction(self): + if self.result_set.getInt("from_me") == self._INCOMING_CALL_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL + + def get_phone_number_from(self): + if self.get_call_direction() == self.INCOMING_CALL: + sender = self.result_set.getString("num") + return Account.Address(sender, sender) + return super(WhatsAppSingleCallLogsParser, self).get_phone_number_from() + + def get_phone_number_to(self): + if self.get_call_direction() == self.OUTGOING_CALL: + to = self.result_set.getString("num") + return Account.Address(to, to) + return super(WhatsAppSingleCallLogsParser, self).get_phone_number_to() + + def get_call_start_date_time(self): + return self.result_set.getLong("timestamp") / 1000 + + def get_call_end_date_time(self): + start = self.get_call_start_date_time() + duration = self.result_set.getInt("duration") + return start + duration + + def get_call_type(self): + if self.result_set.getInt("video_call") == self._VIDEO_CALL_TYPE: + return self.VIDEO_CALL + return self.AUDIO_CALL + + class WhatsAppContactsParser(TskContactsParser): """ Extracts TSK_CONTACT information from the WhatsApp database. @@ -125,14 +269,31 @@ class WhatsAppContactsParser(TskContactsParser): def __init__(self, contact_db): super(WhatsAppContactsParser, self).__init__(contact_db.runQuery( """ - SELECT """ + _get_contacts_formatting() + """ + SELECT jid, + CASE + WHEN WC.number IS NULL THEN WC.jid + WHEN WC.number == "" THEN WC.jid + ELSE WC.number + END number, + CASE + WHEN WC.given_name IS NULL + AND WC.family_name IS NULL + AND WC.display_name IS NULL THEN WC.jid + WHEN WC.given_name IS NULL + AND WC.family_name IS NULL THEN WC.display_name + WHEN WC.given_name IS NULL THEN WC.family_name + WHEN WC.family_name IS NULL THEN WC.given_name + ELSE WC.given_name + || " " + || WC.family_name + END name FROM wa_contacts AS WC """ - ) + ) ) def get_account_name(self): - return self.get_phone() + return self.result_set.getString("jid") def get_contact_name(self): return self.result_set.getString("name") @@ -150,73 +311,87 @@ class WhatsAppMessagesParser(TskMessagesParser): def __init__(self, message_db): super(WhatsAppMessagesParser, self).__init__(message_db.runQuery( """ - SELECT M.data AS content, - """+_get_contacts_formatting()+""", - M.key_from_me AS direction, - M.received_timestamp AS received_datetime, - M.timestamp AS send_datetime - FROM messages AS M - JOIN wadb.wa_contacts AS WC - ON M.key_remote_jid = WC.jid + SELECT M.key_remote_jid AS id, + contact_info.recipients, + key_from_me AS direction, + CASE + WHEN M.data IS NULL THEN "" + ELSE M.data + END AS content, + M.timestamp AS send_timestamp, + M.received_timestamp, + M.remote_resource AS group_sender, + M.media_url As attachment, + M.media_mime_type as attachment_mimetype + FROM (SELECT jid, + recipients + FROM wadb.wa_contacts AS WC + LEFT JOIN (SELECT gjid, + group_concat(CASE + WHEN jid == "" THEN NULL + ELSE jid + END) AS recipients + FROM group_participants + GROUP BY gjid) AS group_map + ON WC.jid = group_map.gjid + GROUP BY jid) AS contact_info + JOIN messages AS M + ON M.key_remote_jid = contact_info.jid """ ) ) self._WHATSAPP_MESSAGE_TYPE = "WhatsApp Message" self._INCOMING_MESSAGE_TYPE = 0 self._OUTGOING_MESSAGE_TYPE = 1 + self._message_db = message_db def get_message_type(self): return self._WHATSAPP_MESSAGE_TYPE def get_phone_number_to(self): - if self.get_message_direction() == self.OUTGOING_MSG: - return Account.Address(self.result_set.getString("number"), - self.result_set.getString("number")) + group = self.result_set.getString("recipients") + if group is not None: + return Account.Address(self.result_set.getString("id"), group) + if self.get_message_direction() == self.OUTGOING: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("id")) return super(WhatsAppMessagesParser, self).get_phone_number_to() def get_phone_number_from(self): - if self.get_message_direction() == self.INCOMING_MSG: - return Account.Address(self.result_set.getString("number"), - self.result_set.getString("number")) + if self.get_message_direction() == self.INCOMING: + group_sender = self.result_set.getString("group_sender") + group = self.result_set.getString("recipients") + if group_sender is not None and group is not None: + return Account.Address(group_sender, group_sender) + else: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("id")) return super(WhatsAppMessagesParser, self).get_phone_number_from() def get_message_direction(self): direction = self.result_set.getInt("direction") if direction == self._INCOMING_MESSAGE_TYPE: - return self.INCOMING_MSG - return self.OUTGOING_MSG + return self.INCOMING + return self.OUTGOING def get_message_date_time(self): #transform from ms to seconds - if self.get_message_direction() == self.OUTGOING_MSG: - return self.result_set.getLong("send_datetime") / 1000 - return self.result_set.getLong("received_datetime") / 1000 + if self.get_message_direction() == self.OUTGOING: + return self.result_set.getLong("send_timestamp") / 1000 + return self.result_set.getLong("received_timestamp") / 1000 def get_message_text(self): - return self.result_set.getString("content") - -def _get_contacts_formatting(): - """ - This function is here to explicitly stress the point that the - formatting routine used in the contacts and messages parsers - should never differ. These fields are used to correlate in Autopsy. - - The SQL statement assumes wa_contacts table is named WC. - """ - - return """ - CASE - WHEN WC.number IS NULL THEN WC.jid - WHEN WC.number == "" THEN WC.jid - ELSE WC.number - END number, - CASE - WHEN WC.given_name IS NULL - AND WC.family_name IS NULL THEN WC.jid - WHEN WC.given_name IS NULL THEN WC.family_name - WHEN WC.family_name IS NULL THEN WC.given_name - ELSE WC.given_name - || " " - || WC.family_name - END name - """ + message = self.result_set.getString("content") + attachment = self.result_set.getString("attachment") + if attachment is not None: + mime_type = self.result_set.getString("attachment_mimetype") + if mime_type is not None: + attachment += "\nMIME type: " + mime_type + return appendAttachmentList(message, [attachment]) + return message + + def get_thread_id(self): + group = self.result_set.getString("recipients") + if group is not None: + return self.result_set.getString("id") + return super(WhatsAppMessagesParser, self).get_thread_id() From e3142149054852bb0170de50e0581f87d9ad81f0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 19:51:50 -0400 Subject: [PATCH 7/9] Brought the code up to date with the api, fixed bugs --- .../android/TskCallLogsParser.py | 16 +- .../android/TskMessagesParser.py | 17 +- InternalPythonModules/android/whatsapp.py | 234 +++++++++++------- 3 files changed, 167 insertions(+), 100 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 763ba3c15f..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,14 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 9ae57c09d6..0582a558c7 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -16,7 +16,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - from java.io import File from java.lang import Class from java.lang import ClassNotFoundException @@ -26,22 +25,29 @@ from java.sql import ResultSet from java.sql import SQLException from java.sql import Statement from java.util.logging import Level +from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser -from general import appendAttachmentList import traceback import general @@ -67,79 +73,128 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): try: contact_dbs = AppSQLiteDB.findAppDatabases(dataSource, "wa.db", True, self._WHATSAPP_PACKAGE_NAME) - message_dbs = AppSQLiteDB.findAppDatabases(dataSource, + calllog_and_message_dbs = AppSQLiteDB.findAppDatabases(dataSource, "msgstore.db", True, self._WHATSAPP_PACKAGE_NAME) #Extract TSK_CONTACT information for contact_db in contact_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, contact_db.getDBFile(), Account.Type.WHATSAPP) + self.parse_contacts(contact_db, helper) - contacts_parser = WhatsAppContactsParser(contact_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() + for calllog_and_message_db in calllog_and_message_dbs: + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + calllog_and_message_db.getDBFile(), Account.Type.WHATSAPP) + calllog_and_message_db.attachDatabase(dataSource, "wa.db", + calllog_and_message_db.getDBFile().getParentPath(), "wadb") + self.parse_calllogs(calllog_and_message_db, helper) + self.parse_messages(calllog_and_message_db, helper) - contact_db.close() - - for message_db in message_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, - message_db.getDBFile(), Account.Type.WHATSAPP) - - message_db.attachDatabase(dataSource, "wa.db", - message_db.getDBFile().getParentPath(), "wadb") - - messages_parser = WhatsAppMessagesParser(message_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() - - group_calllogs_parser = WhatsAppGroupCallLogsParser(message_db) - while group_calllogs_parser.next(): - helper.addCalllog( - group_calllogs_parser.get_call_direction(), - group_calllogs_parser.get_phone_number_from(), - group_calllogs_parser.get_phone_number_to(), - group_calllogs_parser.get_call_start_date_time(), - group_calllogs_parser.get_call_end_date_time(), - group_calllogs_parser.get_call_type() - ) - group_calllogs_parser.close() - - single_calllogs_parser = WhatsAppSingleCallLogsParser(message_db) - while single_calllogs_parser.next(): - helper.addCalllog( - single_calllogs_parser.get_call_direction(), - single_calllogs_parser.get_phone_number_from(), - single_calllogs_parser.get_phone_number_to(), - single_calllogs_parser.get_call_start_date_time(), - single_calllogs_parser.get_call_end_date_time(), - single_calllogs_parser.get_call_type() - ) - single_calllogs_parser.close() - - message_db.close() - except (SQLException, TskCoreException) as ex: - #Error parsing WhatsApp db - self._logger.log(Level.WARNING, "Error parsing WhatsApp Databases", ex) + except NoCurrentCaseException as ex: + #If there is no current case, bail out immediately. + self._logger.log(Level.WARNING, "No case currently open.", ex) self._logger.log(Level.WARNING, traceback.format_exec()) + + #Clean up open file handles. + for contact_db in contact_dbs: + contact_db.close() + + for calllog_and_message_db in calllog_and_message_dbs: + calllog_and_message_db.close() + + def parse_contacts(self, contacts_db, helper): + try: + contacts_parser = WhatsAppContactsParser(contacts_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the whatsapp database for contacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding whatsapp contact artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting contact artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_calllogs(self, calllogs_db, helper): + try: + single_calllogs_parser = WhatsAppSingleCallLogsParser(calllogs_db) + while single_calllogs_parser.next(): + helper.addCalllog( + single_calllogs_parser.get_call_direction(), + single_calllogs_parser.get_phone_number_from(), + single_calllogs_parser.get_phone_number_to(), + single_calllogs_parser.get_call_start_date_time(), + single_calllogs_parser.get_call_end_date_time(), + single_calllogs_parser.get_call_type() + ) + single_calllogs_parser.close() + + group_calllogs_parser = WhatsAppGroupCallLogsParser(calllogs_db) + while group_calllogs_parser.next(): + helper.addCalllog( + group_calllogs_parser.get_call_direction(), + group_calllogs_parser.get_phone_number_from(), + group_calllogs_parser.get_phone_number_to(), + group_calllogs_parser.get_call_start_date_time(), + group_calllogs_parser.get_call_end_date_time(), + group_calllogs_parser.get_call_type() + ) + group_calllogs_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the whatsapp database for calllogs.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding whatsapp calllog artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting calllog artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_messages(self, messages_db, helper): + try: + messages_parser = WhatsAppMessagesParser(messages_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error querying the whatsapp database for contacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, + "Error adding whatsapp contact artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, + "Error posting contact artifact to the blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) class WhatsAppGroupCallLogsParser(TskCallLogsParser): """ @@ -150,17 +205,19 @@ class WhatsAppGroupCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(WhatsAppGroupCallLogsParser, self).__init__(calllog_db.runQuery( """ - SELECT CL.video_call, - CL.timestamp, - CL.duration, - CL.from_me, - J.raw_string as from_num, - group_concat(J.raw_string) AS group_members - FROM call_log_participant_v2 AS CLP - JOIN call_log AS CL - ON CL._id = CLP.call_log_row_id - JOIN jid AS J - ON J._id = CLP.jid_row_id + SELECT CL.video_call, + CL.timestamp, + CL.duration, + CL.from_me, + J1.raw_string AS from_id, + group_concat(J.raw_string) AS group_members + FROM call_log_participant_v2 AS CLP + JOIN call_log AS CL + ON CL._id = CLP.call_log_row_id + JOIN jid AS J + ON J._id = CLP.jid_row_id + JOIN jid as J1 + ON J1._id = CL.jid_row_id GROUP BY CL._id """ ) @@ -176,7 +233,7 @@ class WhatsAppGroupCallLogsParser(TskCallLogsParser): def get_phone_number_from(self): if self.get_call_direction() == self.INCOMING_CALL: - sender = self.result_set.getString("from_num") + sender = self.result_set.getString("from_id") return Account.Address(sender, sender) return super(WhatsAppGroupCallLogsParser, self).get_phone_number_from() @@ -349,12 +406,19 @@ class WhatsAppMessagesParser(TskMessagesParser): return self._WHATSAPP_MESSAGE_TYPE def get_phone_number_to(self): - group = self.result_set.getString("recipients") - if group is not None: - return Account.Address(self.result_set.getString("id"), group) if self.get_message_direction() == self.OUTGOING: + group = self.result_set.getString("recipients") + if group is not None: + group = group.split(",") + + recipients = [] + for token in group: + recipients.append(Account.Address(token, token)) + + return recipients + return Account.Address(self.result_set.getString("id"), - self.result_set.getString("id")) + self.result_set.getString("id")) return super(WhatsAppMessagesParser, self).get_phone_number_to() def get_phone_number_from(self): @@ -387,7 +451,7 @@ class WhatsAppMessagesParser(TskMessagesParser): mime_type = self.result_set.getString("attachment_mimetype") if mime_type is not None: attachment += "\nMIME type: " + mime_type - return appendAttachmentList(message, [attachment]) + return general.appendAttachmentList(message, [attachment]) return message def get_thread_id(self): From bf37509b1d584bd6e030be2b113418a96b153bfb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:16:39 -0400 Subject: [PATCH 8/9] Moved missed attachDatabase refactor --- InternalPythonModules/android/whatsapp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 0582a558c7..5dfa3c8f16 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -89,8 +89,6 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, calllog_and_message_db.getDBFile(), Account.Type.WHATSAPP) - calllog_and_message_db.attachDatabase(dataSource, "wa.db", - calllog_and_message_db.getDBFile().getParentPath(), "wadb") self.parse_calllogs(calllog_and_message_db, helper) self.parse_messages(calllog_and_message_db, helper) @@ -170,6 +168,9 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): def parse_messages(self, messages_db, helper): try: + messages_db.attachDatabase(dataSource, "wa.db", + messages_db.getDBFile().getParentPath(), "wadb") + messages_parser = WhatsAppMessagesParser(messages_db) while messages_parser.next(): helper.addMessage( From 17c8ed02e548240c76c5a88f204e95b06d021d15 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:20:52 -0400 Subject: [PATCH 9/9] more refactoring --- InternalPythonModules/android/whatsapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 5dfa3c8f16..eeb561923a 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -90,7 +90,7 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): current_case.getSleuthkitCase(), self._PARSER_NAME, calllog_and_message_db.getDBFile(), Account.Type.WHATSAPP) self.parse_calllogs(calllog_and_message_db, helper) - self.parse_messages(calllog_and_message_db, helper) + self.parse_messages(dataSource, calllog_and_message_db, helper) except NoCurrentCaseException as ex: #If there is no current case, bail out immediately. @@ -166,7 +166,7 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): "Error posting calllog artifact to the blackboard.", ex) self._logger.log(Level.WARNING, traceback.format_exc()) - def parse_messages(self, messages_db, helper): + def parse_messages(self, dataSource, messages_db, helper): try: messages_db.attachDatabase(dataSource, "wa.db", messages_db.getDBFile().getParentPath(), "wadb")