From 46c77338e6d8f553ab9d582f7a2b8296076b1303 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 6 Jan 2020 15:46:39 -0500 Subject: [PATCH 01/59] 5866 changes to prevent CVT case closed exception --- .../autopsy/communications/FiltersPanel.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java index 61485832b0..c346d0fe62 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java +++ b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java @@ -65,7 +65,6 @@ import org.sleuthkit.datamodel.CommunicationsFilter.AccountTypeFilter; import org.sleuthkit.datamodel.CommunicationsFilter.DateRangeFilter; import org.sleuthkit.datamodel.CommunicationsFilter.DeviceFilter; import org.sleuthkit.datamodel.CommunicationsFilter.MostRecentFilter; -import org.sleuthkit.datamodel.CommunicationsManager; import org.sleuthkit.datamodel.DataSource; import static org.sleuthkit.datamodel.Relationship.Type.CALL_LOG; import static org.sleuthkit.datamodel.Relationship.Type.CONTACT; @@ -129,7 +128,7 @@ final public class FiltersPanel extends JPanel { public FiltersPanel() { initComponents(); - initalizeDeviceAccountType(); + initalizeDeviceAccountType(); deviceRequiredLabel.setVisible(false); accountTypeRequiredLabel.setVisible(false); @@ -149,7 +148,6 @@ final public class FiltersPanel extends JPanel { updateTimeZone(); validationListener = itemEvent -> validateFilters(); - updateFilters(true); UserPreferences.addChangeListener(preferenceChangeEvent -> { if (preferenceChangeEvent.getKey().equals(UserPreferences.DISPLAY_TIMES_IN_LOCAL_TIME) || preferenceChangeEvent.getKey().equals(UserPreferences.TIME_ZONE_FOR_DISPLAYS)) { @@ -239,9 +237,15 @@ final public class FiltersPanel extends JPanel { * Updates the filter widgets to reflect he data sources/types in the case. */ private boolean updateFilters(boolean initialState) { - boolean newAccountType = updateAccountTypeFilter(initialState); - boolean newDeviceFilter = updateDeviceFilter(initialState); - + final SleuthkitCase sleuthkitCase; + try { + sleuthkitCase = Case.getCurrentCaseThrows().getSleuthkitCase(); + } catch (NoCurrentCaseException ex) { + logger.log(Level.WARNING, "Unable to perform filter update, update has been cancelled. Case is closed.", ex); + return false; + } + boolean newAccountType = updateAccountTypeFilter(initialState, sleuthkitCase); + boolean newDeviceFilter = updateDeviceFilter(initialState, sleuthkitCase); // both or either are true, return true; return newAccountType || newDeviceFilter; } @@ -255,10 +259,10 @@ final public class FiltersPanel extends JPanel { //clear the device filter widget when the case changes. devicesMap.clear(); devicesListPane.removeAll(); - - accountTypeMap.clear(); - accountTypeListPane.removeAll(); - + + accountTypeMap.clear(); + accountTypeListPane.removeAll(); + initalizeDeviceAccountType(); }); } @@ -269,7 +273,7 @@ final public class FiltersPanel extends JPanel { IngestManager.getInstance().removeIngestModuleEventListener(ingestListener); IngestManager.getInstance().removeIngestJobEventListener(ingestJobListener); } - + private void initalizeDeviceAccountType() { CheckBoxIconPanel panel = createAccoutTypeCheckBoxPanel(Account.Type.DEVICE, true); accountTypeMap.put(Account.Type.DEVICE, panel.getCheckBox()); @@ -277,17 +281,18 @@ final public class FiltersPanel extends JPanel { } /** - * Populate the Account Types filter widgets + * Populate the Account Types filter widgets. * - * @param selected the initial value for the account type checkbox + * @param selected The initial value for the account type checkbox. + * @param sleuthkitCase The sleuthkit case for containing the account + * information. * * @return True, if a new accountType was found */ - private boolean updateAccountTypeFilter(boolean selected) { + private boolean updateAccountTypeFilter(boolean selected, SleuthkitCase sleuthkitCase) { boolean newOneFound = false; try { - final CommunicationsManager communicationsManager = Case.getCurrentCaseThrows().getSleuthkitCase().getCommunicationsManager(); - List accountTypesInUse = communicationsManager.getAccountTypesInUse(); + List accountTypesInUse = sleuthkitCase.getCommunicationsManager().getAccountTypesInUse(); for (Account.Type type : accountTypesInUse) { @@ -302,10 +307,7 @@ final public class FiltersPanel extends JPanel { } catch (TskCoreException ex) { logger.log(Level.WARNING, "Unable to update to update Account Types Filter", ex); - } catch (NoCurrentCaseException ex) { - logger.log(Level.WARNING, "A case is required to update the account types filter.", ex); } - if (newOneFound) { accountTypeListPane.revalidate(); } @@ -333,17 +335,17 @@ final public class FiltersPanel extends JPanel { } /** - * Populate the devices filter widgets + * Populate the devices filter widgets. * - * @param selected Sets the initial state of device check box + * @param selected Sets the initial state of device check box. + * @param sleuthkitCase The sleuthkit case for containing the data source + * information. * * @return true if a new device was found */ - private boolean updateDeviceFilter(boolean selected) { + private boolean updateDeviceFilter(boolean selected, SleuthkitCase sleuthkitCase) { boolean newOneFound = false; try { - final SleuthkitCase sleuthkitCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - for (DataSource dataSource : sleuthkitCase.getDataSources()) { String dsName = sleuthkitCase.getContentById(dataSource.getId()).getName(); if (devicesMap.containsKey(dataSource.getDeviceId())) { @@ -358,8 +360,6 @@ final public class FiltersPanel extends JPanel { newOneFound = true; } - } catch (NoCurrentCaseException ex) { - logger.log(Level.INFO, "Filter update cancelled. Case is closed."); } catch (TskCoreException tskCoreException) { logger.log(Level.SEVERE, "There was a error loading the datasources for the case.", tskCoreException); } From 55195308d657dac105e6a2d4c9f44bdab920db49 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Fri, 10 Jan 2020 10:37:58 -0500 Subject: [PATCH 02/59] Update ExtractRecycleBin.java Add $RECYCLE.BIN as the directory to look for $I files. --- .../sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index d3af457a1f..fe53dbbd40 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -71,6 +71,8 @@ final class ExtractRecycleBin extends Extract { private static final Logger logger = Logger.getLogger(ExtractRecycleBin.class.getName()); private static final String RECYCLE_BIN_ARTIFACT_NAME = "TSK_RECYCLE_BIN"; //NON-NLS + + private static final String RECYCLE_BIN_DIR_NAME = "$RECYCLE.BIN"; //NON-NLS private static final int V1_FILE_NAME_OFFSET = 24; private static final int V2_FILE_NAME_OFFSET = 28; @@ -127,7 +129,7 @@ final class ExtractRecycleBin extends Extract { // Get the $I files List iFiles; try { - iFiles = fileManager.findFiles(dataSource, "$I%"); //NON-NLS + iFiles = fileManager.findFiles(dataSource, "$I%", RECYCLE_BIN_DIR_NAME); //NON-NLS } catch (TskCoreException ex) { logger.log(Level.WARNING, "Unable to find recycle bin I files.", ex); //NON-NLS return; // No need to continue @@ -321,7 +323,7 @@ final class ExtractRecycleBin extends Extract { return new RecycledFileMetaData(fileSize, timestamp, fileName); } - + /** * Create a map of userids to usernames from the OS Accounts. * From 810ffcd0ac66a7281f830f915dc98ac61c13ecf6 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 14 Jan 2020 10:21:29 -0500 Subject: [PATCH 03/59] Make photorec error message less specific --- .../modules/photoreccarver/PhotoRecCarverFileIngestModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index e98e15a331..392e9587c9 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -324,7 +324,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { return IngestModule.ProcessResult.ERROR; } catch (IOException ex) { totals.totalItemsWithErrors.incrementAndGet(); - logger.log(Level.SEVERE, String.format("Error writing file '%s' (id=%d) to '%s' with the PhotoRec carver.", file.getName(), file.getId(), tempFilePath), ex); // NON-NLS + logger.log(Level.SEVERE, String.format("Error writing or processing file '%s' (id=%d) to '%s' with the PhotoRec carver.", file.getName(), file.getId(), tempFilePath), ex); // NON-NLS MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.error.msg", file.getName())); return IngestModule.ProcessResult.ERROR; } finally { From a20900e996786ce42b45cafea17e19870e861ac9 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 14 Jan 2020 15:12:39 -0500 Subject: [PATCH 04/59] Update ExtractRecycleBin.java Update exceptions caught and thrown. Update comments. Add file and path name to logging message for future debugging if needed. --- .../recentactivity/ExtractRecycleBin.java | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index fe53dbbd40..001b62cc8d 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -23,10 +23,10 @@ package org.sleuthkit.autopsy.recentactivity; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.BufferUnderflowException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -151,7 +151,8 @@ final class ExtractRecycleBin extends Extract { } /** - * Process each individual iFile. + * Process each individual iFile. Each iFile ($I) contains metadata about files that have been deleted. + * Each $I file should have a corresponding $R file which is the actuall deleted file. * * @param context * @param recycleBinArtifactType Module created artifact type @@ -176,7 +177,7 @@ final class ExtractRecycleBin extends Extract { try { metaData = parseIFile(tempFilePath); } catch (IOException ex) { - logger.log(Level.WARNING, String.format("Unable to parse iFile %s", iFile.getName()), ex); //NON-NLS + logger.log(Level.WARNING, String.format("Unable to parse iFile %s", iFile.getParentPath() + iFile.getName()), ex); //NON-NLS // Unable to parse the $I file move onto the next file return; } @@ -268,37 +269,34 @@ final class ExtractRecycleBin extends Extract { } /** - * Parse the $I file. + * Parse the $I file. This file contains metadata information about deleted files * * File format prior to Windows 10: - * - * - * - * - * - * - *
OffsetSizeDescription
08Header
88File Size
168Deleted Timestamp
24520File Name
- * + * Offset Size Description + * 0 8 Header + * 8 8 File Size + * 16 8 Deleted Timestamp + * 24 520 File Name + * * File format Windows 10+ - * - * - * - * - * - * - * - *
OffsetSizeDescription
08Header
88File Size
168Deleted TimeStamp
244File Name Length
28varFile Name
- * + * Offset Size Description + * 0 8 Header + * 8 8 File Size + * 16 8 Deleted TimeStamp + * 24 4 File Name Length + * 28 var File Name + * * For versions of Windows prior to 10, header = 0x01. Windows 10+ header == * 0x02 * * @param iFilePath Path to local copy of file in temp folder * - * @throws FileNotFoundException * @throws IOException */ - private RecycledFileMetaData parseIFile(String iFilePath) throws FileNotFoundException, IOException { - byte[] allBytes = Files.readAllBytes(Paths.get(iFilePath)); + private RecycledFileMetaData parseIFile(String iFilePath) throws IOException { + try { + byte[] allBytes = Files.readAllBytes(Paths.get(iFilePath)); + ByteBuffer byteBuffer = ByteBuffer.wrap(allBytes); byteBuffer.order(ByteOrder.LITTLE_ENDIAN); @@ -322,6 +320,9 @@ final class ExtractRecycleBin extends Extract { String fileName = new String(stringBytes, "UTF-16LE"); //NON-NLS return new RecycledFileMetaData(fileSize, timestamp, fileName); + } catch (BufferUnderflowException | IllegalArgumentException | ArrayIndexOutOfBoundsException ex) { + throw new IOException("Error parsing $I File, file is corrupt or not a valid I$ file"); + } } /** From 3312585178ce830f7f465422bff8ddaa864f33dc Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 14 Jan 2020 16:02:13 -0500 Subject: [PATCH 05/59] Update ExtractRecycleBin.java Fix for codacy. --- .../sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index 001b62cc8d..adc7170482 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -320,8 +320,8 @@ final class ExtractRecycleBin extends Extract { String fileName = new String(stringBytes, "UTF-16LE"); //NON-NLS return new RecycledFileMetaData(fileSize, timestamp, fileName); - } catch (BufferUnderflowException | IllegalArgumentException | ArrayIndexOutOfBoundsException ex) { - throw new IOException("Error parsing $I File, file is corrupt or not a valid I$ file"); + } catch (IOException | BufferUnderflowException | IllegalArgumentException | ArrayIndexOutOfBoundsException ex) { + throw new IOException("Error parsing $I File, file is corrupt or not a valid I$ file", ex); } } From d857c200bba5a95b1b6156dddace51c35befc52b Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 21 Jan 2020 11:58:03 -0500 Subject: [PATCH 06/59] Update AdHocSearchChildFactory.java Moved position of keyword preview from back of map to front so it will display properly. --- .../keywordsearch/AdHocSearchChildFactory.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java index 2b279bfa88..d01e8837c3 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java @@ -176,6 +176,14 @@ class AdHocSearchChildFactory extends ChildFactory { * Get file properties. */ Map properties = new LinkedHashMap<>(); + + /** + * Add a snippet property, if available. + */ + if (hit.hasSnippet()) { + properties.put(TSK_KEYWORD_PREVIEW.getDisplayName(), hit.getSnippet()); + } + Content content; String contentName; try { @@ -196,12 +204,6 @@ class AdHocSearchChildFactory extends ChildFactory { properties.put(LOCATION.toString(), contentName); } - /** - * Add a snippet property, if available. - */ - if (hit.hasSnippet()) { - properties.put(TSK_KEYWORD_PREVIEW.getDisplayName(), hit.getSnippet()); - } String hitName; BlackboardArtifact artifact = null; From a1d95a9fc9b673c5ec729e52b4ce9f646df42368 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 27 Jan 2020 17:20:59 -0500 Subject: [PATCH 07/59] Cleaned up geolocation threading --- .../geolocation/AbstractWaypointFetcher.java | 251 +++++++++++++ .../geolocation/CheckBoxListPanel.java | 4 + .../autopsy/geolocation/GeoFilterPanel.java | 122 +++++-- .../geolocation/GeolocationTopComponent.java | 338 +++++------------- .../autopsy/geolocation/MapPanel.java | 3 - 5 files changed, 441 insertions(+), 277 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java b/Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java new file mode 100755 index 0000000000..a662b410e6 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java @@ -0,0 +1,251 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019-2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.geolocation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationDataException; +import org.sleuthkit.autopsy.geolocation.datamodel.Track; +import org.sleuthkit.autopsy.geolocation.datamodel.Waypoint; +import org.sleuthkit.autopsy.geolocation.datamodel.WaypointBuilder; + +/** + * The business logic for filtering waypoints. + */ +abstract class AbstractWaypointFetcher implements WaypointBuilder.WaypointFilterQueryCallBack { + + private static final Logger logger = Logger.getLogger(AbstractWaypointFetcher.class.getName()); + + private final GeoFilterPanel.GeoFilter filters; + + /** + * Constructs the Waypoint Runner + * + * @param filters + */ + AbstractWaypointFetcher(GeoFilterPanel.GeoFilter filters) { + this.filters = filters; + } + + /** + * Gets the waypoints based in the current GeoFilter. + * + * This function kicks off a process that will send with + * handleFilteredWaypointSet being called. Subclasses must implement + * handleFitleredWayoiintSet to get the final results. + * + * @throws GeoLocationDataException + */ + void getWaypoints() throws GeoLocationDataException { + Case currentCase = Case.getCurrentCase(); + WaypointBuilder.getAllWaypoints(currentCase.getSleuthkitCase(), + filters.getDataSources(), + filters.showAllWaypoints(), + filters.getMostRecentNumDays(), + filters.showWaypointsWithoutTimeStamp(), + this); + + } + + /** + * Called after all of the MapWaypoints are created from all of the + * TSK_GPS_XXX objects. + * + * @param mapWaypoints List of filtered MapWaypoints. + */ + abstract void handleFilteredWaypointSet(Set mapWaypoints); + + @Override + public void process(List waypoints) { + + List tracks = null; + try { + tracks = Track.getTracks(Case.getCurrentCase().getSleuthkitCase(), filters.getDataSources()); + } catch (GeoLocationDataException ex) { + logger.log(Level.WARNING, "Exception thrown while retrieving list of Tracks", ex); + } + + List completeList = createWaypointList(waypoints, tracks); + final Set pointSet = MapWaypoint.getWaypoints(completeList); + + handleFilteredWaypointSet(pointSet); + } + + /** + * Returns a complete list of waypoints including the tracks. Takes into + * account the current filters and includes waypoints as approprate. + * + * @param waypoints List of waypoints + * @param tracks List of tracks + * + * @return A list of waypoints including the tracks based on the current + * filters. + */ + private List createWaypointList(List waypoints, List tracks) { + final List completeList = new ArrayList<>(); + + if (tracks != null) { + Long timeRangeEnd; + Long timeRangeStart; + if (!filters.showAllWaypoints()) { + // Figure out what the most recent time is given the filtered + // waypoints and the tracks. + timeRangeEnd = getMostRecent(waypoints, tracks); + timeRangeStart = timeRangeEnd - (86400 * filters.getMostRecentNumDays()); + + completeList.addAll(getWaypointsInRange(timeRangeStart, timeRangeEnd, waypoints)); + completeList.addAll(getTracksInRange(timeRangeStart, timeRangeEnd, tracks)); + + } else { + completeList.addAll(waypoints); + for (Track track : tracks) { + completeList.addAll(track.getPath()); + } + } + } else { + completeList.addAll(waypoints); + } + + return completeList; + } + + /** + * Return a list of waypoints that fall into the given time range. + * + * @param timeRangeStart start timestamp of range (seconds from java epoch) + * @param timeRangeEnd start timestamp of range (seconds from java epoch) + * @param waypoints List of waypoints to filter. + * + * @return A list of waypoints that fall into the time range. + */ + private List getWaypointsInRange(Long timeRangeStart, Long timeRangeEnd, List waypoints) { + List completeList = new ArrayList<>(); + // Add all of the waypoints that fix into the time range. + if (waypoints != null) { + for (Waypoint point : waypoints) { + Long time = point.getTimestamp(); + if ((time == null && filters.showWaypointsWithoutTimeStamp()) + || (time != null && (time >= timeRangeStart && time <= timeRangeEnd))) { + + completeList.add(point); + } + } + } + return completeList; + } + + /** + * Return a list of waypoints from the given tracks that fall into for + * tracks that fall into the given time range. The track start time will + * used for determining if the whole track falls into the range. + * + * @param timeRangeStart start timestamp of range (seconds from java epoch) + * @param timeRangeEnd start timestamp of range (seconds from java epoch) + * @param tracks Track list. + * + * @return A list of waypoints that that belong to tracks that fall into the + * time range. + */ + private List getTracksInRange(Long timeRangeStart, Long timeRangeEnd, List tracks) { + List completeList = new ArrayList<>(); + if (tracks != null) { + for (Track track : tracks) { + Long trackTime = track.getStartTime(); + + if ((trackTime == null && filters.showWaypointsWithoutTimeStamp()) + || (trackTime != null && (trackTime >= timeRangeStart && trackTime <= timeRangeEnd))) { + + completeList.addAll(track.getPath()); + } + } + } + return completeList; + } + + /** + * Find the latest time stamp in the given list of waypoints. + * + * @param points List of Waypoints, required. + * + * @return The latest time stamp (seconds from java epoch) + */ + private Long findMostRecentTimestamp(List points) { + + Long mostRecent = null; + + for (Waypoint point : points) { + if (mostRecent == null) { + mostRecent = point.getTimestamp(); + } else { + mostRecent = Math.max(mostRecent, point.getTimestamp()); + } + } + + return mostRecent; + } + + /** + * Find the latest time stamp in the given list of tracks. + * + * @param tracks List of Waypoints, required. + * + * @return The latest time stamp (seconds from java epoch) + */ + private Long findMostRecentTracks(List tracks) { + Long mostRecent = null; + + for (Track track : tracks) { + if (mostRecent == null) { + mostRecent = track.getStartTime(); + } else { + mostRecent = Math.max(mostRecent, track.getStartTime()); + } + } + + return mostRecent; + } + + /** + * Returns the "most recent" timestamp amount the list of waypoints and + * track points. + * + * @param points List of Waypoints + * @param tracks List of Tracks + * + * @return Latest time stamp (seconds from java epoch) + */ + private Long getMostRecent(List points, List tracks) { + Long waypointMostRecent = findMostRecentTimestamp(points); + Long trackMostRecent = findMostRecentTracks(tracks); + + if (waypointMostRecent != null && trackMostRecent != null) { + return Math.max(waypointMostRecent, trackMostRecent); + } else if (waypointMostRecent == null && trackMostRecent != null) { + return trackMostRecent; + } else if (waypointMostRecent != null && trackMostRecent == null) { + return waypointMostRecent; + } + + return null; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/CheckBoxListPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/CheckBoxListPanel.java index 0942a9e528..49dfadaacb 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/CheckBoxListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/CheckBoxListPanel.java @@ -67,6 +67,10 @@ final class CheckBoxListPanel extends javax.swing.JPanel { model.removeAllElements(); } + boolean isEmpty() { + return model.isEmpty(); + } + @Override public void setEnabled(boolean enabled) { checkboxList.setEnabled(enabled); diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java index 51238a033e..cce7a6f28a 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java @@ -20,14 +20,19 @@ package org.sleuthkit.autopsy.geolocation; import java.awt.GridBagConstraints; import java.awt.event.ActionListener; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.logging.Level; +import javafx.util.Pair; import javax.swing.ImageIcon; import javax.swing.SpinnerNumberModel; +import javax.swing.SwingWorker; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -38,12 +43,24 @@ import org.sleuthkit.datamodel.TskCoreException; */ class GeoFilterPanel extends javax.swing.JPanel { + final static String INITPROPERTY = "FilterPanelInitCompleted"; + private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(GeoFilterPanel.class.getName()); private final SpinnerNumberModel numberModel; private final CheckBoxListPanel checkboxPanel; + // Make sure to update if + private static final BlackboardArtifact.ARTIFACT_TYPE[] GPS_ARTIFACT_TYPES = { + BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK, + BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_LAST_KNOWN_LOCATION, + BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_ROUTE, + BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_SEARCH, + BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACK, + BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT + }; + /** * Creates new GeoFilterPanel */ @@ -73,7 +90,7 @@ class GeoFilterPanel extends javax.swing.JPanel { gridBagConstraints.insets = new java.awt.Insets(0, 15, 0, 15); add(checkboxPanel, gridBagConstraints); } - + @Override public void setEnabled(boolean enabled) { applyButton.setEnabled(enabled); @@ -84,18 +101,15 @@ class GeoFilterPanel extends javax.swing.JPanel { daysLabel.setEnabled(enabled); daysSpinner.setEnabled(enabled); } - + /** * Update the data source list with the current data sources */ void updateDataSourceList() { - try { - initCheckboxList(); - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Failed to initialize the CheckboxListPane", ex); //NON-NLS - } + DataSourceUpdater updater = new DataSourceUpdater(); + updater.execute(); } - + /** * Clears the data source list. */ @@ -103,6 +117,10 @@ class GeoFilterPanel extends javax.swing.JPanel { checkboxPanel.clearList(); } + boolean hasDataSources() { + return !checkboxPanel.isEmpty(); + } + /** * Adds an actionListener to listen for the filter apply action * @@ -128,26 +146,12 @@ class GeoFilterPanel extends javax.swing.JPanel { if (dataSources.isEmpty()) { throw new GeoLocationUIException(Bundle.GeoFilterPanel_empty_dataSource()); } - return new GeoFilter(allButton.isSelected(), - showWaypointsWOTSCheckBox.isSelected(), - numberModel.getNumber().intValue(), + return new GeoFilter(allButton.isSelected(), + showWaypointsWOTSCheckBox.isSelected(), + numberModel.getNumber().intValue(), dataSources); } - /** - * Initialize the checkbox list panel - * - * @throws TskCoreException - */ - private void initCheckboxList() throws TskCoreException { - final SleuthkitCase sleuthkitCase = Case.getCurrentCase().getSleuthkitCase(); - - for (DataSource dataSource : sleuthkitCase.getDataSources()) { - String dsName = sleuthkitCase.getContentById(dataSource.getId()).getName(); - checkboxPanel.addElement(dsName, dataSource); - } - } - /** * Based on the state of mostRecent radio button Change the state of the cnt * spinner and the time stamp checkbox. @@ -376,4 +380,72 @@ class GeoFilterPanel extends javax.swing.JPanel { } } + /** + * SwingWorker for updating the list of valid data sources. + * + * doInBackground creates a list of Pair objects that contain the + * display name of the data source and the data source object. + */ + final private class DataSourceUpdater extends SwingWorker>, Void> { + + @Override + protected List> doInBackground() throws Exception { + SleuthkitCase sleuthkitCase = Case.getCurrentCase().getSleuthkitCase(); + List> validSources = new ArrayList<>(); + for (DataSource dataSource : sleuthkitCase.getDataSources()) { + if (isGPSDataSource(sleuthkitCase, dataSource)) { + String dsName = sleuthkitCase.getContentById(dataSource.getId()).getName(); + Pair pair = new Pair<>(dsName, dataSource); + validSources.add(pair); + } + } + + return validSources; + } + + /** + * Returns whether or not the given data source has GPS artifacts. + * + * @param sleuthkitCase The current sleuthkitCase + * @param dataSource + * + * @return True if the data source as at least one TSK_GPS_XXXX + * + * @throws TskCoreException + */ + private boolean isGPSDataSource(SleuthkitCase sleuthkitCase, DataSource dataSource) throws TskCoreException { + for (BlackboardArtifact.ARTIFACT_TYPE type : GPS_ARTIFACT_TYPES) { + if (sleuthkitCase.getBlackboardArtifactsTypeCount(type.getTypeID(), dataSource.getId()) > 0) { + return true; + } + } + + return false; + } + + @Override + public void done() { + List> sources = null; + try { + sources = get(); + } catch (InterruptedException | ExecutionException ex) { + Throwable cause = ex.getCause(); + if (cause != null) { + logger.log(Level.SEVERE, cause.getMessage(), cause); + } else { + logger.log(Level.SEVERE, ex.getMessage(), ex); + } + } + + if (sources != null) { + for (Pair source : sources) { + checkboxPanel.addElement(source.getKey(), source.getValue()); + } + } + + GeoFilterPanel.this.firePropertyChange(INITPROPERTY, false, true); + } + + } + } diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java index 17b5f58d34..a2f5cfe381 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java @@ -28,7 +28,6 @@ import java.io.File; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; import java.util.LinkedHashSet; @@ -51,10 +50,6 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.geolocation.GeoFilterPanel.GeoFilter; import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationDataException; -import org.sleuthkit.autopsy.geolocation.datamodel.Track; -import org.sleuthkit.autopsy.geolocation.datamodel.Waypoint; -import org.sleuthkit.autopsy.geolocation.datamodel.WaypointBuilder; -import org.sleuthkit.autopsy.geolocation.datamodel.WaypointBuilder.WaypointFilterQueryCallBack; import org.sleuthkit.autopsy.ingest.IngestManager; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.DATA_ADDED; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -93,7 +88,9 @@ public final class GeolocationTopComponent extends TopComponent { @Messages({ "GLTopComponent_name=Geolocation", - "GLTopComponent_initilzation_error=An error occurred during waypoint initilization. Geolocation data maybe incomplete." + "GLTopComponent_initilzation_error=An error occurred during waypoint initilization. Geolocation data maybe incomplete.", + "GLTopComponent_No_dataSource_message=There are no data sources with Geolocation artifacts found.", + "GLTopComponent_No_dataSource_Title=No Geolocation artifacts found" }) /** @@ -143,7 +140,6 @@ public final class GeolocationTopComponent extends TopComponent { public void actionPerformed(ActionEvent e) { geoFilterPanel.updateDataSourceList(); mapPanel.clearWaypoints(); - updateWaypoints(); showRefreshPanel(false); } }); @@ -157,6 +153,24 @@ public final class GeolocationTopComponent extends TopComponent { } }); + geoFilterPanel.addPropertyChangeListener(GeoFilterPanel.INITPROPERTY, new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + if (geoFilterPanel.hasDataSources()) { + updateWaypoints(); + } else { + geoFilterPanel.setEnabled(false); + setWaypointLoading(false); + JOptionPane.showMessageDialog(GeolocationTopComponent.this, + Bundle.GLTopComponent_No_dataSource_message(), + Bundle.GLTopComponent_No_dataSource_Title(), + JOptionPane.ERROR_MESSAGE); + + } + } + + }); + mapPanel.addPropertyChangeListener(MapPanel.CURRENT_MOUSE_GEOPOSITION, new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { @@ -200,9 +214,6 @@ public final class GeolocationTopComponent extends TopComponent { @Override public void open() { super.open(); - mapPanel.clearWaypoints(); - geoFilterPanel.clearDataSourceList(); - geoFilterPanel.updateDataSourceList(); // Let's make sure we only do this on the first open if (!mapInitalized) { @@ -221,8 +232,12 @@ public final class GeolocationTopComponent extends TopComponent { return; // Doen't set the waypoints. } } + + mapPanel.clearWaypoints(); + geoFilterPanel.clearDataSourceList(); + geoFilterPanel.updateDataSourceList(); mapPanel.setWaypoints(new LinkedHashSet<>()); - updateWaypoints(); + } /** @@ -236,8 +251,8 @@ public final class GeolocationTopComponent extends TopComponent { public void run() { boolean isShowing = false; Component[] comps = mapPanel.getComponents(); - for(Component comp: comps) { - if(comp.equals(refreshPanel)) { + for (Component comp : comps) { + if (comp.equals(refreshPanel)) { isShowing = true; break; } @@ -245,10 +260,10 @@ public final class GeolocationTopComponent extends TopComponent { if (show && !isShowing) { mapPanel.add(refreshPanel, BorderLayout.NORTH); mapPanel.revalidate(); - } else if(!show && isShowing){ + } else if (!show && isShowing) { mapPanel.remove(refreshPanel); mapPanel.revalidate(); - } + } } }); @@ -283,10 +298,61 @@ public final class GeolocationTopComponent extends TopComponent { setWaypointLoading(true); geoFilterPanel.setEnabled(false); - Thread thread = new Thread(new WaypointRunner(filters)); + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + try { + (new WaypointFetcher(filters)).getWaypoints(); + } catch (GeoLocationDataException ex) { + logger.log(Level.SEVERE, "Failed to filter waypoints.", ex); + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + JOptionPane.showMessageDialog(GeolocationTopComponent.this, + Bundle.GeoTopComponent_filter_exception_Title(), + Bundle.GeoTopComponent_filter_exception_msg(), + JOptionPane.ERROR_MESSAGE); + + setWaypointLoading(false); + } + }); + + } + } + + }); thread.start(); } + /** + * Add the filtered set of waypoints to the map and set the various window + * components to their proper state. + * + * @param waypointList + */ + void addWaypointsToMap(Set waypointList) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + // If the list is empty, tell the user + if (waypointList == null || waypointList.isEmpty()) { + mapPanel.clearWaypoints(); + JOptionPane.showMessageDialog(GeolocationTopComponent.this, + Bundle.GeoTopComponent_no_waypoints_returned_Title(), + Bundle.GeoTopComponent_no_waypoints_returned_mgs(), + JOptionPane.INFORMATION_MESSAGE); + setWaypointLoading(false); + geoFilterPanel.setEnabled(true); + return; + } + mapPanel.clearWaypoints(); + mapPanel.setWaypoints(waypointList); + setWaypointLoading(false); + geoFilterPanel.setEnabled(true); + } + }); + } + /** * Show or hide the waypoint loading progress bar. * @@ -423,244 +489,18 @@ public final class GeolocationTopComponent extends TopComponent { // End of variables declaration//GEN-END:variables /** - * A runnable class for getting waypoints based on the current filters. + * Extends AbstractWaypointFetcher to handle the returning of + * the filters set of MapWaypoints. */ - private class WaypointRunner implements Runnable, WaypointFilterQueryCallBack { + final private class WaypointFetcher extends AbstractWaypointFetcher { - private final GeoFilter filters; - - /** - * Constructs the Waypoint Runner - * - * @param filters - */ - WaypointRunner(GeoFilter filters) { - this.filters = filters; + WaypointFetcher(GeoFilter filters) { + super(filters); } @Override - public void run() { - Case currentCase = Case.getCurrentCase(); - try { - WaypointBuilder.getAllWaypoints(currentCase.getSleuthkitCase(), - filters.getDataSources(), - filters.showAllWaypoints(), - filters.getMostRecentNumDays(), - filters.showWaypointsWithoutTimeStamp(), - this); - - } catch (GeoLocationDataException ex) { - logger.log(Level.SEVERE, "Failed to filter waypoints.", ex); - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - JOptionPane.showMessageDialog(GeolocationTopComponent.this, - Bundle.GeoTopComponent_filter_exception_Title(), - Bundle.GeoTopComponent_filter_exception_msg(), - JOptionPane.ERROR_MESSAGE); - - setWaypointLoading(false); - } - }); - } - } - - @Override - public void process(List waypoints) { - - List tracks = null; - try { - tracks = Track.getTracks(Case.getCurrentCase().getSleuthkitCase(), filters.getDataSources()); - } catch (GeoLocationDataException ex) { - logger.log(Level.WARNING, "Exception thrown while retrieving list of Tracks", ex); - } - - List completeList = createWaypointList(waypoints, tracks); - final Set pointSet = MapWaypoint.getWaypoints(completeList); - - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - // If the list is empty, tell the user and do not change - // the visible waypoints. - if (completeList == null || completeList.isEmpty()) { - mapPanel.clearWaypoints(); - JOptionPane.showMessageDialog(GeolocationTopComponent.this, - Bundle.GeoTopComponent_no_waypoints_returned_Title(), - Bundle.GeoTopComponent_no_waypoints_returned_mgs(), - JOptionPane.INFORMATION_MESSAGE); - setWaypointLoading(false); - geoFilterPanel.setEnabled(true); - return; - } - mapPanel.clearWaypoints(); - mapPanel.setWaypoints(pointSet); - setWaypointLoading(false); - geoFilterPanel.setEnabled(true); - } - }); - } - - /** - * Returns a complete list of waypoints including the tracks. Takes into - * account the current filters and includes waypoints as approprate. - * - * @param waypoints List of waypoints - * @param tracks List of tracks - * - * @return A list of waypoints including the tracks based on the current - * filters. - */ - private List createWaypointList(List waypoints, List tracks) { - final List completeList = new ArrayList<>(); - - if (tracks != null) { - Long timeRangeEnd; - Long timeRangeStart; - if (!filters.showAllWaypoints()) { - // Figure out what the most recent time is given the filtered - // waypoints and the tracks. - timeRangeEnd = getMostRecent(waypoints, tracks); - timeRangeStart = timeRangeEnd - (86400 * filters.getMostRecentNumDays()); - - completeList.addAll(getWaypointsInRange(timeRangeStart, timeRangeEnd, waypoints)); - completeList.addAll(getTracksInRange(timeRangeStart, timeRangeEnd, tracks)); - - } else { - completeList.addAll(waypoints); - for (Track track : tracks) { - completeList.addAll(track.getPath()); - } - } - } else { - completeList.addAll(waypoints); - } - - return completeList; - } - - /** - * Return a list of waypoints that fall into the given time range. - * - * @param timeRangeStart start timestamp of range (seconds from java - * epoch) - * @param timeRangeEnd start timestamp of range (seconds from java - * epoch) - * @param waypoints List of waypoints to filter. - * - * @return A list of waypoints that fall into the time range. - */ - private List getWaypointsInRange(Long timeRangeStart, Long timeRangeEnd, List waypoints) { - List completeList = new ArrayList<>(); - // Add all of the waypoints that fix into the time range. - if (waypoints != null) { - for (Waypoint point : waypoints) { - Long time = point.getTimestamp(); - if ((time == null && filters.showWaypointsWithoutTimeStamp()) - || (time != null && (time >= timeRangeStart && time <= timeRangeEnd))) { - - completeList.add(point); - } - } - } - return completeList; - } - - /** - * Return a list of waypoints from the given tracks that fall into for - * tracks that fall into the given time range. The track start time will - * used for determining if the whole track falls into the range. - * - * @param timeRangeStart start timestamp of range (seconds from java - * epoch) - * @param timeRangeEnd start timestamp of range (seconds from java - * epoch) - * @param tracks Track list. - * - * @return A list of waypoints that that belong to tracks that fall into - * the time range. - */ - private List getTracksInRange(Long timeRangeStart, Long timeRangeEnd, List tracks) { - List completeList = new ArrayList<>(); - if (tracks != null) { - for (Track track : tracks) { - Long trackTime = track.getStartTime(); - - if ((trackTime == null && filters.showWaypointsWithoutTimeStamp()) - || (trackTime != null && (trackTime >= timeRangeStart && trackTime <= timeRangeEnd))) { - - completeList.addAll(track.getPath()); - } - } - } - return completeList; - } - - /** - * Find the latest time stamp in the given list of waypoints. - * - * @param points List of Waypoints, required. - * - * @return The latest time stamp (seconds from java epoch) - */ - private Long findMostRecentTimestamp(List points) { - - Long mostRecent = null; - - for (Waypoint point : points) { - if (mostRecent == null) { - mostRecent = point.getTimestamp(); - } else { - mostRecent = Math.max(mostRecent, point.getTimestamp()); - } - } - - return mostRecent; - } - - /** - * Find the latest time stamp in the given list of tracks. - * - * @param tracks List of Waypoints, required. - * - * @return The latest time stamp (seconds from java epoch) - */ - private Long findMostRecentTracks(List tracks) { - Long mostRecent = null; - - for (Track track : tracks) { - if (mostRecent == null) { - mostRecent = track.getStartTime(); - } else { - mostRecent = Math.max(mostRecent, track.getStartTime()); - } - } - - return mostRecent; - } - - /** - * Returns the "most recent" timestamp amount the list of waypoints and - * track points. - * - * @param points List of Waypoints - * @param tracks List of Tracks - * - * @return Latest time stamp (seconds from java epoch) - */ - private Long getMostRecent(List points, List tracks) { - Long waypointMostRecent = findMostRecentTimestamp(points); - Long trackMostRecent = findMostRecentTracks(tracks); - - if (waypointMostRecent != null && trackMostRecent != null) { - return Math.max(waypointMostRecent, trackMostRecent); - } else if (waypointMostRecent == null && trackMostRecent != null) { - return trackMostRecent; - } else if (waypointMostRecent != null && trackMostRecent == null) { - return waypointMostRecent; - } - - return null; + void handleFilteredWaypointSet(Set mapWaypoints) { + addWaypointsToMap(mapWaypoints); } } } diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java index 4d286467b8..aa528f77de 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; -import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.logging.Level; @@ -58,7 +57,6 @@ import org.jxmapviewer.viewer.DefaultTileFactory; import org.jxmapviewer.viewer.GeoPosition; import org.jxmapviewer.viewer.TileFactory; import org.jxmapviewer.viewer.TileFactoryInfo; -import org.jxmapviewer.viewer.Waypoint; import org.jxmapviewer.viewer.WaypointPainter; import org.jxmapviewer.viewer.WaypointRenderer; import org.openide.util.NbBundle.Messages; @@ -69,7 +67,6 @@ import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationDataException; import org.sleuthkit.datamodel.TskCoreException; import javax.imageio.ImageIO; import javax.swing.SwingUtilities; -import org.jxmapviewer.viewer.DefaultWaypointRenderer; /** * The map panel. This panel contains the jxmapviewer MapViewer From 15d112e3e8340c39c9f779e6e249d43e5ead6e8e Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 28 Jan 2020 13:49:14 -0500 Subject: [PATCH 08/59] Check for empty database paths. --- .../experimental/configuration/SharedConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java index f9dba08034..0fa7f156ee 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java @@ -1243,9 +1243,9 @@ public class SharedConfiguration { HashDbManager hashDbManager = HashDbManager.getInstance(); hashDbManager.loadLastSavedConfiguration(); for (HashDbManager.HashDb hashDb : hashDbManager.getAllHashSets()) { - if (hashDb.hasIndexOnly()) { + if (hashDb.hasIndexOnly() && (!hashDb.getIndexPath().isEmpty())) { results.add(hashDb.getIndexPath()); - } else { + } else if (!hashDb.getDatabasePath().isEmpty()) { results.add(hashDb.getDatabasePath()); } } From 0ef999ade23fc0dc9509d63ef26ace1e7abe7d55 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Mon, 3 Feb 2020 13:29:46 -0500 Subject: [PATCH 09/59] 5907: Update legacy modules - Removed getConnectionMetadata() api and added columnExists() & tableExists() --- .../autopsy/coreutils/AppSQLiteDB.java | 92 +++++++++++++++++-- InternalPythonModules/android/calllog.py | 64 ++++++------- InternalPythonModules/android/contact.py | 8 +- 3 files changed, 119 insertions(+), 45 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java index ffdb6cf861..f75618b59b 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.coreutils; import java.io.File; import java.io.IOException; import java.sql.Connection; -import java.sql.DatabaseMetaData; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; @@ -288,15 +287,94 @@ public final class AppSQLiteDB { } /** - * Returns connection meta data. - * - * @return DatabaseMetaData - * @throws SQLException + * Checks if a column exists in a table. + * + * @param tableName name of the table + * @param columnName column name to check + * + * @return true if the column exists, false otherwise + * @throws TskCoreException */ - public DatabaseMetaData getConnectionMetadata() throws SQLException { - return connection.getMetaData(); + public boolean columnExists(String tableName, String columnName) throws TskCoreException { + + boolean columnExists = false; + Statement colExistsStatement = null; + ResultSet resultSet = null; + try { + colExistsStatement = connection.createStatement(); + String tableInfoQuery = "PRAGMA table_info(%s)"; //NON-NLS + resultSet = colExistsStatement.executeQuery(String.format(tableInfoQuery, tableName)); + while (resultSet.next()) { + if (resultSet.getString("name").equalsIgnoreCase(columnName)) { + columnExists = true; + break; + } + } + } catch (SQLException ex) { + throw new TskCoreException("Error checking if column " + columnName + "exists ", ex); + } finally { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException ex2) { + logger.log(Level.WARNING, "Failed to close resultset after checking column", ex2); + } + } + if (colExistsStatement != null) { + try { + colExistsStatement.close(); + } catch (SQLException ex2) { + logger.log(Level.SEVERE, "Error closing Statement", ex2); //NON-NLS + } + } + } + return columnExists; } + /** + * Checks if a table exists in the case database. + * + * @param tableName name of the table to check + * + * @return true if the table exists, false otherwise + * @throws TskCoreException + */ + public boolean tableExists(String tableName) throws TskCoreException { + + boolean tableExists = false; + Statement tableExistsStatement = null; + ResultSet resultSet = null; + try { + + tableExistsStatement = connection.createStatement(); + resultSet = tableExistsStatement.executeQuery("SELECT name FROM sqlite_master WHERE type='table'"); //NON-NLS + while (resultSet.next()) { + if (resultSet.getString("name").equalsIgnoreCase(tableName)) { //NON-NLS + tableExists = true; + break; + } + } + } catch (SQLException ex) { + throw new TskCoreException("Error checking if table " + tableName + "exists ", ex); + } finally { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException ex2) { + logger.log(Level.WARNING, "Failed to close resultset after checking table", ex2); + } + } + if (tableExistsStatement != null) { + try { + tableExistsStatement.close(); + } catch (SQLException ex2) { + logger.log(Level.SEVERE, "Error closing Statement", ex2); //NON-NLS + } + } + } + return tableExists; + } + /** * Searches for a meta file associated with the give SQLite database. If * found, it copies this file into the temp directory of the current case. diff --git a/InternalPythonModules/android/calllog.py b/InternalPythonModules/android/calllog.py index 6f89d703ab..71c5f112ad 100644 --- a/InternalPythonModules/android/calllog.py +++ b/InternalPythonModules/android/calllog.py @@ -92,39 +92,41 @@ class CallLogAnalyzer(general.AndroidComponentAnalyzer): for tableName in CallLogAnalyzer._tableNames: try: - resultSet = callLogDb.runQuery("SELECT number, date, duration, type, name FROM " + tableName + " ORDER BY date DESC;") - self._logger.log(Level.INFO, "Reading call log from table {0} in db {1}", [tableName, callLogDb.getDBFile().getName()]) - if resultSet is not None: - while resultSet.next(): - direction = "" - callerId = None - calleeId = None - - timeStamp = resultSet.getLong("date") / 1000 - - number = resultSet.getString("number") - duration = resultSet.getLong("duration") # duration of call is in seconds - name = resultSet.getString("name") # name of person dialed or called. None if unregistered - - calltype = resultSet.getInt("type") - if calltype == 1 or calltype == 3: - direction = CommunicationDirection.INCOMING - callerId = number - elif calltype == 2 or calltype == 5: - direction = CommunicationDirection.OUTGOING - calleeId = number - else: - direction = CommunicationDirection.UNKNOWN + tableFound = callLogDb.tableExists(tableName) + if tableFound: + resultSet = callLogDb.runQuery("SELECT number, date, duration, type, name FROM " + tableName + " ORDER BY date DESC;") + self._logger.log(Level.INFO, "Reading call log from table {0} in db {1}", [tableName, callLogDb.getDBFile().getName()]) + if resultSet is not None: + while resultSet.next(): + direction = "" + callerId = None + calleeId = None + timeStamp = resultSet.getLong("date") / 1000 + + number = resultSet.getString("number") + duration = resultSet.getLong("duration") # duration of call is in seconds + name = resultSet.getString("name") # name of person dialed or called. None if unregistered - ## add a call log - if callerId is not None or calleeId is not None: - callLogArtifact = callLogDbHelper.addCalllog( direction, - callerId, - calleeId, - timeStamp, ## start time - timeStamp + duration * 1000, ## end time - CallMediaType.AUDIO) + calltype = resultSet.getInt("type") + if calltype == 1 or calltype == 3: + direction = CommunicationDirection.INCOMING + callerId = number + elif calltype == 2 or calltype == 5: + direction = CommunicationDirection.OUTGOING + calleeId = number + else: + direction = CommunicationDirection.UNKNOWN + + + ## add a call log + if callerId is not None or calleeId is not None: + callLogArtifact = callLogDbHelper.addCalllog( direction, + callerId, + calleeId, + timeStamp, ## start time + timeStamp + duration * 1000, ## end time + CallMediaType.AUDIO) except SQLException as ex: self._logger.log(Level.WARNING, "Error processing query result for Android messages.", ex) diff --git a/InternalPythonModules/android/contact.py b/InternalPythonModules/android/contact.py index 745440b6da..8144890134 100644 --- a/InternalPythonModules/android/contact.py +++ b/InternalPythonModules/android/contact.py @@ -102,13 +102,7 @@ class ContactAnalyzer(general.AndroidComponentAnalyzer): # get display_name, mimetype(email or phone number) and data1 (phonenumber or email address depending on mimetype) # sorted by name, so phonenumber/email would be consecutive for a person if they exist. # check if contacts.name_raw_contact_id exists. Modify the query accordingly. - columnFound = False - metadata = contactDb.getConnectionMetadata() - columnListResultSet = metadata.getColumns(None, None, "contacts", None) - while columnListResultSet.next(): - if columnListResultSet.getString("COLUMN_NAME") == "name_raw_contact_id": - columnFound = True - break + columnFound = contactDb.columnExists("contacts", "name_raw_contact_id") if columnFound: resultSet = contactDb.runQuery( "SELECT mimetype, data1, name_raw_contact.display_name AS display_name \n" From 2ad2f1fb76eb510a4226ece00b430ea07f0e01e9 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Tue, 4 Feb 2020 14:51:36 -0500 Subject: [PATCH 10/59] Refactored and added comments. No logic changes --- .../recentactivity/Bundle.properties-MERGED | 18 +- .../autopsy/recentactivity/Chrome.java | 2 +- .../recentactivity/ChromeCacheExtractor.java | 534 +++++++++--------- 3 files changed, 287 insertions(+), 267 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index 4535bf7685..18deff87f4 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -1,15 +1,22 @@ cannotBuildXmlParser=Unable to build XML parser: cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml: cannotParseXml=Unable to parse XML file: -ChromeCacheExtract_adding_extracted_files_msg=Adding %d extracted files for analysis. +Chrome.getBookmark.errMsg.errAnalyzeFile={0}: Error while trying to analyze file: {1} +ChromeCacheExtract_adding_artifacts_msg=Chrome Cache: Adding %d artifacts for analysis. +ChromeCacheExtract_adding_extracted_files_msg=Chrome Cache: Adding %d extracted files for analysis. +ChromeCacheExtract_loading_files_msg=Chrome Cache: Loading files from %s. ChromeCacheExtractor.moduleName=ChromeCacheExtractor +# {0} - module name +# {1} - row number +# {2} - table length +# {3} - cache path ChromeCacheExtractor.progressMsg={0}: Extracting cache entry {1} of {2} entries from {3} DataSourceUsage_AndroidMedia=Android Media Card DataSourceUsage_DJU_Drone_DAT=DJI Internal SD Card DataSourceUsage_FlashDrive=Flash Drive -# {0} - OS name DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0}) DataSourceUsageAnalyzer.parentModuleName=Recent Activity +Extract.dbConn.errMsg.failedToQueryDb={0}: Failed to query database. Extract.indexError.message=Failed to index artifact for keyword search. Extract.noOpenCase.errMsg=No open case available. ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history @@ -18,6 +25,11 @@ ExtractEdge_process_errMsg_errGettingWebCacheFiles=Error trying to retrieving Ed ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer ExtractEdge_process_errMsg_webcacheFail=Failure processing Microsoft Edge WebCacheV01.dat file +ExtractIE.getBookmark.ere.noSpace=RecentActivity +ExtractIE.getBookmark.errMsg.errPostingBookmarks=Error posting Internet Explorer Bookmark artifacts. +ExtractIE.getCookie.errMsg.errPostingCookies=Error posting Internet Explorer Cookie artifacts. +ExtractIE.getHistory.errMsg.errPostingHistory=Error posting Internet Explorer History artifacts. +Extractor.errPostingArtifacts=Error posting {0} artifacts to the blackboard. ExtractOs.androidOs.label=Android ExtractOs.androidVolume.label=OS Drive (Android) ExtractOs.debianLinuxOs.label=Linux (Debian) @@ -84,7 +96,7 @@ Chrome.getLogin.errMsg.errAnalyzingFiles={0}: Error while trying to analyze file Chrome.getAutofill.errMsg.errGettingFiles=Error when trying to get Chrome Web Data files. Chrome.getAutofill.errMsg.errAnalyzingFiles={0}: Error while trying to analyze file:{1} ExtractIE.moduleName.text=Internet Explorer -ExtractIE.getBookmark.errMsg.errGettingBookmarks={0}: Error getting Internet Explorer Bookmarks. +ExtractIE.getBookmark.errMsg.errGettingBookmarks=Error getting Internet Explorer Bookmarks. ExtractIE.parentModuleName.noSpace=RecentActivity ExtractIE.parentModuleName=Recent Activity ExtractIE.getURLFromIEBmkFile.errMsg={0}: Error parsing IE bookmark File {1} diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java index 49ac9911fe..f194c6b69e 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java @@ -142,7 +142,7 @@ class Chrome extends Extract { progressBar.progress(Bundle.Progress_Message_Chrome_Cache()); ChromeCacheExtractor chromeCacheExtractor = new ChromeCacheExtractor(dataSource, context, progressBar); - chromeCacheExtractor.getCaches(); + chromeCacheExtractor.processCaches(); } /** diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java index 12c4208d89..9969fbb486 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java @@ -73,6 +73,18 @@ import org.sleuthkit.datamodel.TskException; * * We extract cache entries, create derived files if needed, * and record the URL. + * + * CACHE BASICS (https://www.chromium.org/developers/design-documents/network-stack/disk-cache) + * - A cached item is broken up into segments (header, payload, etc.). The segments are not stored together. + * - Each user has a cache folder in AppData\Local\Google\Chrome\User Data\Default\Cache + * - Each folder has three kinds of files + * -- index: This is the main file. It has one entry for every cached item. You can start from here and work you way to the various data_x and f_XXX files that contain segments. + * -- data_X: These files are containers for small segments and other supporting data (such as the cache entry) + * -- f_XXXX: If the cached data cannot fit into a slot in data_X, it will be saved to its + * own f_XXXX file. These could be compressed if the data being sent was compressed. + * These are referred to as "External Files" in the below code. + * - A CacheAddress embeds information about which file something is stored in. This address is used in several structures to make it easy to abstract out where data is stored. + * - General Flow: index file -> process Cache Entry in data_X file -> process segment in data_X or f_XXX. */ final class ChromeCacheExtractor { @@ -100,22 +112,22 @@ final class ChromeCacheExtractor { private FileManager fileManager; // A file table to cache copies of index and data_n files. - private final Map fileCopyCache = new HashMap<>(); + private final Map fileCopyCache = new HashMap<>(); // A file table to cache the f_* files. private final Map externalFilesTable = new HashMap<>(); /** - * Encapsulates abstract file for a cache file as well as a temp file copy - * that can be accessed as a random access file. + * Allows methods to use data in an AbstractFile in a variety of + * ways. As a ByteBuffer, AbstractFile, etc. A local copy of the file + * backs the ByteBuffer. */ - final class CacheFileCopy { - + final class FileWrapper { private final AbstractFile abstractFile; private final RandomAccessFile fileCopy; private final ByteBuffer byteBuffer; - CacheFileCopy (AbstractFile abstractFile, RandomAccessFile fileCopy, ByteBuffer buffer ) { + FileWrapper (AbstractFile abstractFile, RandomAccessFile fileCopy, ByteBuffer buffer ) { this.abstractFile = abstractFile; this.fileCopy = fileCopy; this.byteBuffer = buffer; @@ -174,13 +186,13 @@ final class ChromeCacheExtractor { } /** - * Initializes the module to extract cache from a specific folder. + * Resets the internal caches and temp folders in between processing each user cache folder * - * @param cachePath - path where cache files are found + * @param cachePath - path (in data source) of the cache being processed * * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ - private void resetForNewFolder(String cachePath) throws IngestModuleException { + private void resetForNewCacheFolder(String cachePath) throws IngestModuleException { fileCopyCache.clear(); externalFilesTable.clear(); @@ -206,7 +218,7 @@ final class ChromeCacheExtractor { */ private void cleanup () { - for (Entry entry : this.fileCopyCache.entrySet()) { + for (Entry entry : this.fileCopyCache.entrySet()) { Path tempFilePath = Paths.get(RAImageIngestModule.getRATempPath(currentCase, moduleName), entry.getKey() ); try { entry.getValue().getFileCopy().getChannel().close(); @@ -246,7 +258,7 @@ final class ChromeCacheExtractor { * A data source may have multiple Chrome user profiles and caches. * */ - void getCaches() { + void processCaches() { try { moduleInit(); @@ -257,18 +269,18 @@ final class ChromeCacheExtractor { } // Find and process the cache folders. There could be one per user - List indexFiles; try { - indexFiles = findCacheIndexFiles(); + // Identify each cache folder by searching for the index files in each + List indexFiles = findIndexFiles(); - // Process each of the caches + // Process each of the cache folders for (AbstractFile indexFile: indexFiles) { if (context.dataSourceIngestIsCancelled()) { return; } - processCacheIndexFile(indexFile); + processCacheFolder(indexFile); } } catch (TskCoreException ex) { @@ -278,62 +290,77 @@ final class ChromeCacheExtractor { } @Messages({ - "ChromeCacheExtract_adding_extracted_files_msg=Adding %d extracted files for analysis." + "ChromeCacheExtract_adding_extracted_files_msg=Chrome Cache: Adding %d extracted files for analysis.", + "ChromeCacheExtract_adding_artifacts_msg=Chrome Cache: Adding %d artifacts for analysis.", + "ChromeCacheExtract_loading_files_msg=Chrome Cache: Loading files from %s." }) /** * Processes a user's cache and creates corresponding artifacts and derived files. + * Will ultimately process the f_XXXX and data_X files in the folder. * - * @param cacheIndexFile Cache index file for a given user + * @param indexFile Index file that is located in a user's cache folder */ - private void processCacheIndexFile(AbstractFile indexAbstractFile) { + private void processCacheFolder(AbstractFile indexFile) { - String cachePath = indexAbstractFile.getParentPath(); - Optional indexFileCopy; + String cacheFolderName = indexFile.getParentPath(); + Optional indexFileWrapper; + + /* + * The first part of this method is all about finding the needed files in the cache + * folder and making internal copies/caches of them so that we can later process them + * and effeciently look them up. + */ try { - resetForNewFolder(cachePath); + progressBar.progress(String.format(Bundle.ChromeCacheExtract_loading_files_msg(), cacheFolderName)); + resetForNewCacheFolder(cacheFolderName); // @@@ This is little ineffecient because we later in this call search for the AbstractFile that we currently have - indexFileCopy = this.getCacheFileCopy(indexAbstractFile.getName(), cachePath); - if (!indexFileCopy.isPresent()) { - String msg = String.format("Failed to find copy cache index file %s", indexAbstractFile.getUniquePath()); + // Load the index file into the caches + indexFileWrapper = findDataOrIndexFile(indexFile.getName(), cacheFolderName); + if (!indexFileWrapper.isPresent()) { + String msg = String.format("Failed to find copy cache index file %s", indexFile.getUniquePath()); logger.log(Level.WARNING, msg); return; } - // load the data files. We do this now to load them into the cache + // load the data files into the internal cache. We do this because we often + // jump in between the various data_X files resolving segments for (int i = 0; i < 4; i ++) { - Optional dataFile = findAndCopyCacheFile(String.format("data_%1d",i), cachePath ); + Optional dataFile = findDataOrIndexFile(String.format("data_%1d",i), cacheFolderName ); if (!dataFile.isPresent()) { return; } } // find all f_* files in a single query and load them into the cache - findExternalFiles(cachePath); + // we do this here so that it is a single query instead of hundreds of individual ones + findExternalFiles(cacheFolderName); } catch (TskCoreException | IngestModuleException ex) { - String msg = "Failed to find cache files in path " + cachePath; //NON-NLS + String msg = "Failed to find cache files in path " + cacheFolderName; //NON-NLS logger.log(Level.WARNING, msg, ex); return; } - - // parse the index file - logger.log(Level.INFO, "{0}- Now reading Cache index file from path {1}", new Object[]{moduleName, cachePath }); //NON-NLS + /* + * Now the analysis begins. We parse the index file and that drives parsing entries + * from data_X or f_XXXX files. + */ + logger.log(Level.INFO, "{0}- Now reading Cache index file from path {1}", new Object[]{moduleName, cacheFolderName }); //NON-NLS List derivedFiles = new ArrayList<>(); - Collection sourceArtifacts = new ArrayList<>(); - Collection webCacheArtifacts = new ArrayList<>(); - - ByteBuffer indexFileROBuffer = indexFileCopy.get().getByteBuffer(); + Collection artifactsAdded = new ArrayList<>(); + + ByteBuffer indexFileROBuffer = indexFileWrapper.get().getByteBuffer(); IndexFileHeader indexHdr = new IndexFileHeader(indexFileROBuffer); // seek past the header indexFileROBuffer.position(INDEXFILE_HDR_SIZE); - // Process each address in the table + /* Cycle through index and get the CacheAddress for each CacheEntry. Process each entry + * to extract data, add artifacts, etc. from the f_XXXX and data_x files */ for (int i = 0; i < indexHdr.getTableLen(); i++) { if (context.dataSourceIngestIsCancelled()) { @@ -341,13 +368,13 @@ final class ChromeCacheExtractor { return; } - CacheAddress addr = new CacheAddress(indexFileROBuffer.getInt() & UINT32_MASK, cachePath); + CacheAddress addr = new CacheAddress(indexFileROBuffer.getInt() & UINT32_MASK, cacheFolderName); if (addr.isInitialized()) { - progressBar.progress( NbBundle.getMessage(this.getClass(), + progressBar.progress(NbBundle.getMessage(this.getClass(), "ChromeCacheExtractor.progressMsg", - moduleName, i, indexHdr.getTableLen(), cachePath) ); + moduleName, i, indexHdr.getTableLen(), cacheFolderName) ); try { - List addedFiles = this.processCacheEntry(addr, sourceArtifacts, webCacheArtifacts); + List addedFiles = processCacheEntry(addr, artifactsAdded); derivedFiles.addAll(addedFiles); } catch (TskCoreException | IngestModuleException ex) { @@ -361,19 +388,19 @@ final class ChromeCacheExtractor { return; } - progressBar.progress(String.format(Bundle.ChromeCacheExtract_adding_extracted_files_msg(), derivedFiles.size())); + // notify listeners of new files and schedule for analysis + progressBar.progress(String.format(Bundle.ChromeCacheExtract_adding_extracted_files_msg(), derivedFiles.size())); derivedFiles.forEach((derived) -> { services.fireModuleContentEvent(new ModuleContentEvent(derived)); }); - context.addFilesToJob(derivedFiles); + // notify listeners about new artifacts + progressBar.progress(String.format(Bundle.ChromeCacheExtract_adding_artifacts_msg(), artifactsAdded.size())); Blackboard blackboard = currentCase.getSleuthkitCase().getBlackboard(); - try { - blackboard.postArtifacts(sourceArtifacts, moduleName); - blackboard.postArtifacts(webCacheArtifacts, moduleName); + blackboard.postArtifacts(artifactsAdded, moduleName); } catch (Blackboard.BlackboardException ex) { logger.log(Level.WARNING, String.format("Failed to post cacheIndex artifacts "), ex); //NON-NLS } @@ -382,49 +409,47 @@ final class ChromeCacheExtractor { } /** - * Gets the cache entry at the specified address. + * Processes the cache entry that is stored at the given address. A CacheEntry is + * located in a data_X file and stores information about where the various segments + * for a given cached entry are located. * * Extracts the files if needed and adds as derived files, creates artifacts * - * @param cacheEntryAddress cache entry address - * @param associatedObjectArtifacts any associated object artifacts created are added to this collection - * @param webCacheArtifacts any web cache artifacts created are added to this collection + * @param cacheAddress Address where CacheEntry is located (from index file) + * @param artifactsAdded any artifact that was added * * @return Optional derived file, is a derived file is added for the given entry */ - private List processCacheEntry(CacheAddress cacheEntryAddress, Collection associatedObjectArtifacts, Collection webCacheArtifacts ) throws TskCoreException, IngestModuleException { + private List processCacheEntry(CacheAddress cacheAddress, Collection artifactsAdded ) throws TskCoreException, IngestModuleException { List derivedFiles = new ArrayList<>(); - // get the path to the corresponding data_X file - String dataFileName = cacheEntryAddress.getFilename(); - String cachePath = cacheEntryAddress.getCachePath(); - - - Optional cacheEntryFile = this.getCacheFileCopy(dataFileName, cachePath); - if (!cacheEntryFile.isPresent()) { - String msg = String.format("Failed to get cache entry at address %s", cacheEntryAddress); //NON-NLS + // get the path to the corresponding data_X file for the cache entry + String cacheEntryFileName = cacheAddress.getFilename(); + String cachePath = cacheAddress.getCachePath(); + + Optional cacheEntryFileOptional = findDataOrIndexFile(cacheEntryFileName, cachePath); + if (!cacheEntryFileOptional.isPresent()) { + String msg = String.format("Failed to find data file %s", cacheEntryFileName); //NON-NLS throw new IngestModuleException(msg); } + // Load the entry to get its metadata, segments, etc. + CacheEntry cacheEntry = new CacheEntry(cacheAddress, cacheEntryFileOptional.get() ); + List dataSegments = cacheEntry.getDataSegments(); - // Get the cache entry and its data segments - CacheEntry cacheEntry = new CacheEntry(cacheEntryAddress, cacheEntryFile.get() ); - - List dataEntries = cacheEntry.getData(); // Only process the first payload data segment in each entry // first data segement has the HTTP headers, 2nd is the payload - if (dataEntries.size() < 2) { + if (dataSegments.size() < 2) { return derivedFiles; } - CacheData dataSegment = dataEntries.get(1); + CacheDataSegment dataSegment = dataSegments.get(1); - - // name of the file that was downloaded and cached (or data_X if it was saved into there) - String cachedFileName = dataSegment.getAddress().getFilename(); - Optional cachedFileAbstractFile = this.findCacheFile(cachedFileName, cachePath); - if (!cachedFileAbstractFile.isPresent()) { - logger.log(Level.WARNING, "Error finding file: " + cachePath + "/" + cachedFileName); //NON-NLS + // Name where segment is located (could be diffrent from where entry was located) + String segmentFileName = dataSegment.getCacheAddress().getFilename(); + Optional segmentFileAbstractFile = findAbstractFile(segmentFileName, cachePath); + if (!segmentFileAbstractFile.isPresent()) { + logger.log(Level.WARNING, "Error finding segment file: " + cachePath + "/" + segmentFileName); //NON-NLS return derivedFiles; } @@ -433,114 +458,92 @@ final class ChromeCacheExtractor { isBrotliCompressed = true; } - // setup some attributes for later use - BlackboardAttribute urlAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, - moduleName, - ((cacheEntry.getKey() != null) ? cacheEntry.getKey() : "")); - BlackboardAttribute createTimeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, - moduleName, - cacheEntry.getCreationTime()); - BlackboardAttribute httpHeaderAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HEADERS, - moduleName, - cacheEntry.getHTTPHeaders()); - - Collection webCacheAttributes = new ArrayList<>(); - webCacheAttributes.add(urlAttr); - webCacheAttributes.add(createTimeAttr); - webCacheAttributes.add(httpHeaderAttr); - - // add artifacts to the f_XXX file - if (dataSegment.isInExternalFile() ) { - try { - - BlackboardArtifact webCacheArtifact = cacheEntryFile.get().getAbstractFile().newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE); - if (webCacheArtifact != null) { - webCacheArtifact.addAttributes(webCacheAttributes); + // Make artifacts around the cached item and extract data from data_X file + try { + AbstractFile cachedItemFile; // + /* If the cached data is in a f_XXXX file, we only need to make artifacts. */ + if (dataSegment.isInExternalFile() ) { + cachedItemFile = segmentFileAbstractFile.get(); + } + /* If the data is in a data_X file, we need to extract it out and then make the similar artifacts */ + else { - // Add path of f_* file as attribute - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, - moduleName, - cachedFileAbstractFile.get().getUniquePath())); - - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, - moduleName, cachedFileAbstractFile.get().getId())); - - webCacheArtifacts.add(webCacheArtifact); - - BlackboardArtifact associatedObjectArtifact = cachedFileAbstractFile.get().newArtifact(ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT); - if (associatedObjectArtifact != null) { - associatedObjectArtifact.addAttribute( - new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, - moduleName, webCacheArtifact.getArtifactID())); - associatedObjectArtifacts.add(associatedObjectArtifact); - } - } - - if (isBrotliCompressed) { - cachedFileAbstractFile.get().setMIMEType(BROTLI_MIMETYPE); - cachedFileAbstractFile.get().save(); - } - } catch (TskException ex) { - logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS - } - } - // extract the embedded data to a derived file and create artifacts - else { - - // Data segments in "data_x" files are saved in individual files and added as derived files - String filename = dataSegment.save(); - String relPathname = getRelOutputFolderName() + dataSegment.getAddress().getCachePath() + filename; - try { + // Data segments in "data_x" files are saved in individual files and added as derived files + String filename = dataSegment.save(); + String relPathname = getRelOutputFolderName() + dataSegment.getCacheAddress().getCachePath() + filename; + + // @@@ We should batch these up and do them in one big insert / transaction DerivedFile derivedFile = fileManager.addDerivedFile(filename, relPathname, dataSegment.getDataLength(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), // TBD true, - cachedFileAbstractFile.get(), + segmentFileAbstractFile.get(), "", moduleName, VERSION_NUMBER, "", TskData.EncodingType.NONE); - - BlackboardArtifact webCacheArtifact = cacheEntryFile.get().getAbstractFile().newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE); - if (webCacheArtifact != null) { - webCacheArtifact.addAttributes(webCacheAttributes); - - // Add path of derived file as attribute - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, - moduleName, - derivedFile.getUniquePath())); - - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, - moduleName, derivedFile.getId())); - - webCacheArtifacts.add(webCacheArtifact); - - BlackboardArtifact associatedObjectArtifact = derivedFile.newArtifact(ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT); - if (associatedObjectArtifact != null) { - associatedObjectArtifact.addAttribute( - new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, - moduleName, webCacheArtifact.getArtifactID())); - associatedObjectArtifacts.add(associatedObjectArtifact); - } - } - - if (isBrotliCompressed) { - derivedFile.setMIMEType(BROTLI_MIMETYPE); - derivedFile.save(); - } - derivedFiles.add(derivedFile); - } catch (TskException ex) { - logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS + cachedItemFile = derivedFile; } + + addArtifacts(cacheEntry, cacheEntryFileOptional.get().getAbstractFile(), cachedItemFile, artifactsAdded); + + // Tika doesn't detect these types. So, make sure they have the correct MIME type */ + if (isBrotliCompressed) { + cachedItemFile.setMIMEType(BROTLI_MIMETYPE); + cachedItemFile.save(); + } + + } catch (TskException ex) { + logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS } return derivedFiles; } + /** + * Add artifacts for a given cached item + * + * @param cacheEntry Entry item came from + * @param cacheEntryFile File that stored the cache entry + * @param cachedItemFile File that stores the cached data (Either a derived file or f_XXXX file) + * @param artifactsAdded List of artifacts that were added by this call + * @throws TskCoreException + */ + private void addArtifacts(CacheEntry cacheEntry, AbstractFile cacheEntryFile, AbstractFile cachedItemFile, Collection artifactsAdded) throws TskCoreException { + + // Create a TSK_WEB_CACHE entry with the parent as data_X file that had the cache entry + BlackboardArtifact webCacheArtifact = cacheEntryFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE); + if (webCacheArtifact != null) { + Collection webAttr = new ArrayList<>(); + webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, + moduleName, + ((cacheEntry.getKey() != null) ? cacheEntry.getKey() : ""))); + webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, + moduleName, cacheEntry.getCreationTime())); + webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HEADERS, + moduleName, cacheEntry.getHTTPHeaders())); + webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, + moduleName, cachedItemFile.getUniquePath())); + webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, + moduleName, cachedItemFile.getId())); + webCacheArtifact.addAttributes(webAttr); + artifactsAdded.add(webCacheArtifact); + + // Create a TSK_ASSOCIATED_OBJECT on the f_XXX or derived file file back to the CACHE entry + BlackboardArtifact associatedObjectArtifact = cachedItemFile.newArtifact(ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT); + if (associatedObjectArtifact != null) { + associatedObjectArtifact.addAttribute( + new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, + moduleName, webCacheArtifact.getArtifactID())); + artifactsAdded.add(associatedObjectArtifact); + } + } + } + /** * Finds all the f_* files in the specified path, and fills them in the * effFilesTable, so that subsequent searches are fast. @@ -557,26 +560,27 @@ final class ChromeCacheExtractor { } } /** - * Finds abstract file for cache file with a specified name. + * Finds a file with a given name in a given cache folder * First checks in the file tables. * * @param cacheFileName * @return Optional abstract file * @throws TskCoreException */ - private Optional findCacheFile(String cacheFileName, String cachePath) throws TskCoreException { + private Optional findAbstractFile(String cacheFileName, String cacheFolderName) throws TskCoreException { // see if it is cached - String fileTableKey = cachePath + cacheFileName; + String fileTableKey = cacheFolderName + cacheFileName; if (cacheFileName.startsWith("f_") && externalFilesTable.containsKey(fileTableKey)) { return Optional.of(externalFilesTable.get(fileTableKey)); } + if (fileCopyCache.containsKey(fileTableKey)) { return Optional.of(fileCopyCache.get(fileTableKey).getAbstractFile()); } - List cacheFiles = fileManager.findFiles(dataSource, cacheFileName, cachePath); //NON-NLS + List cacheFiles = fileManager.findFiles(dataSource, cacheFileName, cacheFolderName); //NON-NLS if (!cacheFiles.isEmpty()) { for (AbstractFile abstractFile: cacheFiles ) { if (abstractFile.getUniquePath().trim().endsWith(DEFAULT_CACHE_PATH_STR)) { @@ -590,57 +594,51 @@ final class ChromeCacheExtractor { } /** - * Finds abstract file(s) for a cache file with the specified name. + * Finds the "index" file that exists in each user's cache. This is used to + * enumerate all of the caches on the system. * - * @return list of abstract files matching the specified file name + * @return list of index files in Chrome cache folders * @throws TskCoreException */ - private List findCacheIndexFiles() throws TskCoreException { + private List findIndexFiles() throws TskCoreException { return fileManager.findFiles(dataSource, "index", DEFAULT_CACHE_PATH_STR); //NON-NLS } + /** - * Returns CacheFileCopy for the specified file from the file table. - * Find the file and creates a copy if it isn't already in the table. - * - * @param cacheFileName Name of file - * @param cachePath Parent path of file - * @return CacheFileCopy + * Finds the specified data or index cache file under the specified path. + * The FileWrapper is easier to parse than a raw AbstractFile. + * Will save the file to an internal cache. For the f_XXXX files, use + * findAbstractFile(). + * + * @param cacheFileName Name file file + * @param cacheFolderName Name of user's cache folder + * @return Cache file copy * @throws TskCoreException - */ - private Optional getCacheFileCopy(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { + */ + private Optional findDataOrIndexFile(String cacheFileName, String cacheFolderName) throws TskCoreException, IngestModuleException { // Check if the file is already in the cache - String fileTableKey = cachePath + cacheFileName; + String fileTableKey = cacheFolderName + cacheFileName; if (fileCopyCache.containsKey(fileTableKey)) { return Optional.of(fileCopyCache.get(fileTableKey)); } - return findAndCopyCacheFile(cacheFileName, cachePath); - } - - /** - * Finds the specified cache file under the specified path, and makes a temporary copy. - * - * @param cacheFileName - * @return Cache file copy - * @throws TskCoreException - */ - private Optional findAndCopyCacheFile(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { - - Optional cacheFileOptional = findCacheFile(cacheFileName, cachePath); - if (!cacheFileOptional.isPresent()) { + // Use Autopsy to get the AbstractFile + Optional abstractFileOptional = findAbstractFile(cacheFileName, cacheFolderName); + if (!abstractFileOptional.isPresent()) { return Optional.empty(); } - + + // Wrap the file so that we can get the ByteBuffer later. + // @@@ BC: I think this should nearly all go into FileWrapper and be done lazily and perhaps based on size. + // Many of the files are small enough to keep in memory for the ByteBuffer // write the file to disk so that we can have a memory-mapped ByteBuffer - // @@@ NOTE: I"m not sure this is needed. These files are small enough and we could probably just load them into - // a byte[] for ByteBuffer. - AbstractFile cacheFile = cacheFileOptional.get(); + AbstractFile cacheFile = abstractFileOptional.get(); RandomAccessFile randomAccessFile = null; - String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + cachePath + cacheFile.getName(); //NON-NLS + String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + cacheFolderName + cacheFile.getName(); //NON-NLS try { File newFile = new File(tempFilePathname); ContentUtils.writeToFile(cacheFile, newFile, context::dataSourceIngestIsCancelled); @@ -651,13 +649,13 @@ final class ChromeCacheExtractor { (int) roChannel.size()); cacheFileROBuf.order(ByteOrder.nativeOrder()); - CacheFileCopy cacheFileCopy = new CacheFileCopy(cacheFile, randomAccessFile, cacheFileROBuf ); + FileWrapper cacheFileWrapper = new FileWrapper(cacheFile, randomAccessFile, cacheFileROBuf ); if (!cacheFileName.startsWith("f_")) { - fileCopyCache.put(cachePath + cacheFileName, cacheFileCopy); + fileCopyCache.put(cacheFolderName + cacheFileName, cacheFileWrapper); } - return Optional.of(cacheFileCopy); + return Optional.of(cacheFileWrapper); } catch (IOException ex) { @@ -699,7 +697,7 @@ final class ChromeCacheExtractor { lastFile = indexFileROBuf.getInt(); indexFileROBuf.position(indexFileROBuf.position()+4); // this_id - indexFileROBuf.position(indexFileROBuf.position()+4); // stats cache address + indexFileROBuf.position(indexFileROBuf.position()+4); // stats cache cacheAddress tableLen = indexFileROBuf.getInt(); } @@ -745,7 +743,7 @@ final class ChromeCacheExtractor { } /** - * Cache file type enum - as encoded the address + * Cache file type enum - as encoded the cacheAddress */ enum CacheFileTypeEnum { EXTERNAL, @@ -761,26 +759,29 @@ final class ChromeCacheExtractor { /** - * Encapsulates Cache address. - * - * CacheAddress is a unsigned 32 bit number + * Google defines the notion of a CacheAddress that spans the various + * files in the cache. The 32-bit number embeds which file and offset + * the address is in. + + * The below defines what each bit means. A 1 means the bit is used + * for that value. * * Header: - * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit - * 0111 0000 0000 0000 0000 0000 0000 0000 : file type - * - * If separate file: - * 0000 1111 1111 1111 1111 1111 1111 1111 : file# 0 - 268,435,456 (2^28) - * - * If block file: - * 0000 1100 0000 0000 0000 0000 0000 0000 : reserved bits - * 0000 0011 0000 0000 0000 0000 0000 0000 : number of contiguous blocks 1-4 - * 0000 0000 1111 1111 0000 0000 0000 0000 : file selector 0 - 255 - * 0000 0000 0000 0000 1111 1111 1111 1111 : block# 0 - 65,535 (2^16) + * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit + * 0111 0000 0000 0000 0000 0000 0000 0000 : file type + * + * If external file: (i.e. f_XXXX) + * 0000 1111 1111 1111 1111 1111 1111 1111 : file# 0 - 268,435,456 (2^28) + * + * If block file: (i.e. data_X) + * 0000 1100 0000 0000 0000 0000 0000 0000 : reserved bits + * 0000 0011 0000 0000 0000 0000 0000 0000 : number of contiguous blocks 1-4 + * 0000 0000 1111 1111 0000 0000 0000 0000 : file selector 0 - 255 + * 0000 0000 0000 0000 1111 1111 1111 1111 : block# 0 - 65,535 (2^16) * */ final class CacheAddress { - // sundry constants to parse the bit fields in address + // sundry constants to parse the bit fields private static final long ADDR_INITIALIZED_MASK = 0x80000000l; private static final long FILE_TYPE_MASK = 0x70000000; private static final long FILE_TYPE_OFFSET = 28; @@ -801,11 +802,18 @@ final class ChromeCacheExtractor { private final String cachePath; + /** + * + * @param uint32 Encoded address + * @param cachePath Folder that index file was located in + */ CacheAddress(long uint32, String cachePath) { uint32CacheAddr = uint32; this.cachePath = cachePath; + + // analyze the int fileTypeEnc = (int)(uint32CacheAddr & FILE_TYPE_MASK) >> FILE_TYPE_OFFSET; fileType = CacheFileTypeEnum.values()[fileTypeEnc]; @@ -930,33 +938,33 @@ final class ChromeCacheExtractor { * * A data segment may be compressed - GZIP and BRotli are the two commonly used methods. */ - final class CacheData { + final class CacheDataSegment { private int length; - private final CacheAddress address; + private final CacheAddress cacheAddress; private CacheDataTypeEnum type; private boolean isHTTPHeaderHint; - private CacheFileCopy cacheFileCopy = null; + private FileWrapper cacheFileCopy = null; private byte[] data = null; private String httpResponse; private final Map httpHeaders = new HashMap<>(); - CacheData(CacheAddress cacheAdress, int len) { - this(cacheAdress, len, false); + CacheDataSegment(CacheAddress cacheAddress, int len) { + this(cacheAddress, len, false); } - CacheData(CacheAddress cacheAdress, int len, boolean isHTTPHeader ) { + CacheDataSegment(CacheAddress cacheAddress, int len, boolean isHTTPHeader ) { this.type = CacheDataTypeEnum.UNKNOWN; this.length = len; - this.address = cacheAdress; + this.cacheAddress = cacheAddress; this.isHTTPHeaderHint = isHTTPHeader; } boolean isInExternalFile() { - return address.isInExternalFile(); + return cacheAddress.isInExternalFile(); } boolean hasHTTPHeaders() { @@ -1006,13 +1014,13 @@ final class ChromeCacheExtractor { } // Don't extract data from external files. - if (!address.isInExternalFile() ) { + if (!cacheAddress.isInExternalFile() ) { - cacheFileCopy = getCacheFileCopy(address.getFilename(), address.getCachePath()).get(); + cacheFileCopy = findDataOrIndexFile(cacheAddress.getFilename(), cacheAddress.getCachePath()).get(); this.data = new byte [length]; ByteBuffer buf = cacheFileCopy.getByteBuffer(); - int dataOffset = DATAFILE_HDR_SIZE + address.getStartBlock() * address.getBlockSize(); + int dataOffset = DATAFILE_HDR_SIZE + cacheAddress.getStartBlock() * cacheAddress.getBlockSize(); buf.position(dataOffset); buf.get(data, 0, length); @@ -1095,8 +1103,8 @@ final class ChromeCacheExtractor { return type; } - CacheAddress getAddress() { - return address; + CacheAddress getCacheAddress() { + return cacheAddress; } @@ -1111,13 +1119,13 @@ final class ChromeCacheExtractor { String save() throws TskCoreException, IngestModuleException { String fileName; - if (address.isInExternalFile()) { - fileName = address.getFilename(); + if (cacheAddress.isInExternalFile()) { + fileName = cacheAddress.getFilename(); } else { - fileName = String.format("%s__%08x", address.getFilename(), address.getUint32CacheAddr()); + fileName = String.format("%s__%08x", cacheAddress.getFilename(), cacheAddress.getUint32CacheAddr()); } - String filePathName = getAbsOutputFolderName() + address.getCachePath() + fileName; + String filePathName = getAbsOutputFolderName() + cacheAddress.getCachePath() + fileName; save(filePathName); return fileName; @@ -1199,7 +1207,7 @@ final class ChromeCacheExtractor { // int32 state; // Current state. // uint64 creation_time; // int32 key_len; -// CacheAddr long_key; // Optional address of a long key. +// CacheAddr long_key; // Optional cacheAddress of a long key. // int32 data_size[4]; // We can store up to 4 data streams for each // CacheAddr data_addr[4]; // entry. // uint32 flags; // Any combination of EntryFlags. @@ -1217,7 +1225,7 @@ final class ChromeCacheExtractor { private static final int MAX_KEY_LEN = 256-24*4; private final CacheAddress selfAddress; - private final CacheFileCopy cacheFileCopy; + private final FileWrapper cacheFileCopy; private final long hash; private final CacheAddress nextAddress; @@ -1230,17 +1238,17 @@ final class ChromeCacheExtractor { private final long creationTime; private final int keyLen; - private final CacheAddress longKeyAddresses; // address of the key, if the key is external to the entry + private final CacheAddress longKeyAddresses; // cacheAddress of the key, if the key is external to the entry - private final int dataSizes[]; - private final CacheAddress dataAddresses[]; - private List dataList; + private final int[] dataSegmentSizes; + private final CacheAddress[] dataSegmentIndexFileEntries; + private List dataSegments; private final long flags; private String key; // Key may be found within the entry or may be external - CacheEntry(CacheAddress cacheAdress, CacheFileCopy cacheFileCopy ) { + CacheEntry(CacheAddress cacheAdress, FileWrapper cacheFileCopy ) { this.selfAddress = cacheAdress; this.cacheFileCopy = cacheFileCopy; @@ -1270,14 +1278,14 @@ final class ChromeCacheExtractor { uint32 = fileROBuf.getInt() & UINT32_MASK; longKeyAddresses = (uint32 != 0) ? new CacheAddress(uint32, selfAddress.getCachePath()) : null; - dataList = null; - dataSizes= new int[4]; + dataSegments = null; + dataSegmentSizes= new int[4]; for (int i = 0; i < 4; i++) { - dataSizes[i] = fileROBuf.getInt(); + dataSegmentSizes[i] = fileROBuf.getInt(); } - dataAddresses = new CacheAddress[4]; + dataSegmentIndexFileEntries = new CacheAddress[4]; for (int i = 0; i < 4; i++) { - dataAddresses[i] = new CacheAddress(fileROBuf.getInt() & UINT32_MASK, selfAddress.getCachePath()); + dataSegmentIndexFileEntries[i] = new CacheAddress(fileROBuf.getInt() & UINT32_MASK, selfAddress.getCachePath()); } flags = fileROBuf.getInt() & UINT32_MASK; @@ -1293,7 +1301,7 @@ final class ChromeCacheExtractor { if (longKeyAddresses != null) { // Key is stored outside of the entry try { - CacheData data = new CacheData(longKeyAddresses, this.keyLen, true); + CacheDataSegment data = new CacheDataSegment(longKeyAddresses, this.keyLen, true); key = data.getDataString(); } catch (TskCoreException | IngestModuleException ex) { logger.log(Level.WARNING, String.format("Failed to get external key from address %s", longKeyAddresses)); //NON-NLS @@ -1315,7 +1323,7 @@ final class ChromeCacheExtractor { } } - public CacheAddress getAddress() { + public CacheAddress getCacheAddress() { return selfAddress; } @@ -1323,7 +1331,7 @@ final class ChromeCacheExtractor { return hash; } - public CacheAddress getNextAddress() { + public CacheAddress getNextCacheAddress() { return nextAddress; } @@ -1359,20 +1367,20 @@ final class ChromeCacheExtractor { * @throws TskCoreException * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ - public List getData() throws TskCoreException, IngestModuleException { + public List getDataSegments() throws TskCoreException, IngestModuleException { - if (dataList == null) { - dataList = new ArrayList<>(); + if (dataSegments == null) { + dataSegments = new ArrayList<>(); for (int i = 0; i < 4; i++) { - if (dataSizes[i] > 0) { - CacheData cacheData = new CacheData(dataAddresses[i], dataSizes[i], true ); + if (dataSegmentSizes[i] > 0) { + CacheDataSegment cacheData = new CacheDataSegment(dataSegmentIndexFileEntries[i], dataSegmentSizes[i], true ); cacheData.extract(); - dataList.add(cacheData); + dataSegments.add(cacheData); } } } - return dataList; + return dataSegments; } /** @@ -1383,10 +1391,10 @@ final class ChromeCacheExtractor { * @return true if the entry has HTTP headers */ boolean hasHTTPHeaders() { - if ((dataList == null) || dataList.isEmpty()) { + if ((dataSegments == null) || dataSegments.isEmpty()) { return false; } - return dataList.get(0).hasHTTPHeaders(); + return dataSegments.get(0).hasHTTPHeaders(); } /** @@ -1396,11 +1404,11 @@ final class ChromeCacheExtractor { * @return header value, null if not found */ String getHTTPHeader(String key) { - if ((dataList == null) || dataList.isEmpty()) { + if ((dataSegments == null) || dataSegments.isEmpty()) { return null; } // First data segment has the HTTP headers, if any - return dataList.get(0).getHTTPHeader(key); + return dataSegments.get(0).getHTTPHeader(key); } /** @@ -1409,11 +1417,11 @@ final class ChromeCacheExtractor { * @return header value, null if not found */ String getHTTPHeaders() { - if ((dataList == null) || dataList.isEmpty()) { + if ((dataSegments == null) || dataSegments.isEmpty()) { return null; } // First data segment has the HTTP headers, if any - return dataList.get(0).getHTTPHeaders(); + return dataSegments.get(0).getHTTPHeaders(); } /** @@ -1449,11 +1457,11 @@ final class ChromeCacheExtractor { (nextAddress != null) ? nextAddress.toString() : "None")); for (int i = 0; i < 4; i++) { - if (dataSizes[i] > 0) { + if (dataSegmentSizes[i] > 0) { sb.append(String.format("\n\tData %d: cache address = %s, Data = %s", - i, dataAddresses[i].toString(), - (dataList != null) - ? dataList.get(i).toString() + i, dataSegmentIndexFileEntries[i].toString(), + (dataSegments != null) + ? dataSegments.get(i).toString() : "Data not retrived yet.")); } } From 687f7c56cb16f95f6d957b485580b12c96fe484e Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Tue, 4 Feb 2020 22:55:58 -0500 Subject: [PATCH 11/59] 6036: reduce cache processing time. Ignore associated objects and minor change along with TSK change to get data source faster --- .../textextractors/ArtifactTextExtractor.java | 12 +----------- .../autopsy/keywordsearch/SolrSearchService.java | 3 ++- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/ArtifactTextExtractor.java b/Core/src/org/sleuthkit/autopsy/textextractors/ArtifactTextExtractor.java index ff806845f1..43c0112f1a 100644 --- a/Core/src/org/sleuthkit/autopsy/textextractors/ArtifactTextExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/textextractors/ArtifactTextExtractor.java @@ -46,16 +46,6 @@ class ArtifactTextExtractor implements TextExtractor { // "content" string to be indexed. StringBuilder artifactContents = new StringBuilder(); - Content dataSource = null; - try { - dataSource = artifact.getDataSource(); - } catch (TskCoreException tskCoreException) { - throw new InitReaderException("Unable to get datasource for artifact: " + artifact.toString(), tskCoreException); - } - if (dataSource == null) { - throw new InitReaderException("Datasource was null for artifact: " + artifact.toString()); - } - try { for (BlackboardAttribute attribute : artifact.getAttributes()) { artifactContents.append(attribute.getAttributeType().getDisplayName()); @@ -67,7 +57,7 @@ class ArtifactTextExtractor implements TextExtractor { // in the Autopsy datamodel. switch (attribute.getValueType()) { case DATETIME: - artifactContents.append(ContentUtils.getStringTime(attribute.getValueLong(), dataSource)); + artifactContents.append(ContentUtils.getStringTime(attribute.getValueLong(), artifact)); break; default: artifactContents.append(attribute.getDisplayString()); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 94dd567c5e..daaccc91ce 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -457,7 +457,8 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { @Subscribe void handleNewArtifacts(Blackboard.ArtifactsPostedEvent event) { for (BlackboardArtifact artifact : event.getArtifacts()) { - if (artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) { //don't index KWH artifacts. + if ((artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) && // don't index KWH bc it's based on existing indexed text + (artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT.getTypeID())){ //don't index AO bc it has only an artifact ID - no useful text try { index(artifact); } catch (TskCoreException ex) { From cc7ed224ffbae0a190e3645dea130132928811e6 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Thu, 6 Feb 2020 16:25:06 -0500 Subject: [PATCH 12/59] Interim commit - isolated scheme creation code from DBSettings classes and put them in the new RdmsCentralRepoSchemaFactory. --- .../datamodel/PostgresCentralRepo.java | 9 +- .../PostgresCentralRepoSettings.java | 289 +------- .../datamodel/RdbmsCentralRepo.java | 33 +- .../RdbmsCentralRepoSchemaFactory.java | 696 ++++++++++++++++++ .../datamodel/SqliteCentralRepo.java | 9 +- .../datamodel/SqliteCentralRepoSettings.java | 319 +------- .../optionspanel/EamDbSettingsDialog.form | 4 +- .../optionspanel/EamDbSettingsDialog.java | 23 +- 8 files changed, 757 insertions(+), 625 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java index 42772a9bd3..93c82c7bdf 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java @@ -131,7 +131,10 @@ final class PostgresCentralRepo extends RdbmsCentralRepo { CentralRepoDbUtil.closeConnection(conn); } - dbSettings.insertDefaultDatabaseContent(); + + //dbSettings.insertDefaultDatabaseContent(); + RdbmsCentralRepoSchemaFactory centralRepoSchemaFactory = new RdbmsCentralRepoSchemaFactory(CentralRepoPlatforms.POSTGRESQL); + centralRepoSchemaFactory.insertDefaultDatabaseContent(); } /** @@ -209,6 +212,10 @@ final class PostgresCentralRepo extends RdbmsCentralRepo { return CONFLICT_CLAUSE; } + @Override + protected Connection getEphemeralConnection() { + return this.dbSettings.getEphemeralConnection(false); + } /** * Gets an exclusive lock (if applicable). Will return the lock if * successful, null if unsuccessful because locking isn't supported, and diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java index 9b5b013540..10204e0ffe 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java @@ -162,7 +162,7 @@ public final class PostgresCentralRepoSettings { * * @return Connection or null. */ - private Connection getEphemeralConnection(boolean usePostgresDb) { + Connection getEphemeralConnection(boolean usePostgresDb) { Connection conn; try { String url = getConnectionURL(usePostgresDb); @@ -290,308 +290,25 @@ public final class PostgresCentralRepoSettings { } - /** - * Initialize the database schema. - * - * Requires valid connectionPool. - * - * This method is called from within connect(), so we cannot call connect() - * to get a connection. This method is called after setupConnectionPool(), - * so it is safe to assume that a valid connectionPool exists. The - * implementation of connect() is synchronized, so we can safely use the - * connectionPool object directly. - */ - public boolean initializeDatabaseSchema() { - // The "id" column is an alias for the built-in 64-bit int "rowid" column. - // It is autoincrementing by default and must be of type "integer primary key". - // We've omitted the autoincrement argument because we are not currently - // using the id value to search for specific rows, so we do not care - // if a rowid is re-used after an existing rows was previously deleted. - StringBuilder createOrganizationsTable = new StringBuilder(); - createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); - createOrganizationsTable.append("id SERIAL PRIMARY KEY,"); - createOrganizationsTable.append("org_name text NOT NULL,"); - createOrganizationsTable.append("poc_name text NOT NULL,"); - createOrganizationsTable.append("poc_email text NOT NULL,"); - createOrganizationsTable.append("poc_phone text NOT NULL,"); - createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); - createOrganizationsTable.append(")"); + - // NOTE: The organizations will only have a small number of rows, so - // an index is probably not worthwhile. - StringBuilder createCasesTable = new StringBuilder(); - createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); - createCasesTable.append("id SERIAL PRIMARY KEY,"); - createCasesTable.append("case_uid text NOT NULL,"); - createCasesTable.append("org_id integer,"); - createCasesTable.append("case_name text NOT NULL,"); - createCasesTable.append("creation_date text NOT NULL,"); - createCasesTable.append("case_number text,"); - createCasesTable.append("examiner_name text,"); - createCasesTable.append("examiner_email text,"); - createCasesTable.append("examiner_phone text,"); - createCasesTable.append("notes text,"); - createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)"); - createCasesTable.append(")"); - // NOTE: when there are few cases in the cases table, these indices may not be worthwhile - String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; - String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - StringBuilder createReferenceSetsTable = new StringBuilder(); - createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); - createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); - createReferenceSetsTable.append("org_id integer NOT NULL,"); - createReferenceSetsTable.append("set_name text NOT NULL,"); - createReferenceSetsTable.append("version text NOT NULL,"); - createReferenceSetsTable.append("known_status integer NOT NULL,"); - createReferenceSetsTable.append("read_only boolean NOT NULL,"); - createReferenceSetsTable.append("type integer NOT NULL,"); - createReferenceSetsTable.append("import_date text NOT NULL,"); - createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); - createReferenceSetsTable.append(")"); + - String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; - // Each "%s" will be replaced with the relevant reference_TYPE table name. - StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); - createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createReferenceTypesTableTemplate.append("id SERIAL PRIMARY KEY,"); - createReferenceTypesTableTemplate.append("reference_set_id integer,"); - createReferenceTypesTableTemplate.append("value text NOT NULL,"); - createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); - createReferenceTypesTableTemplate.append("comment text,"); - createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE (reference_set_id, value),"); - createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createReferenceTypesTableTemplate.append(")"); - // Each "%s" will be replaced with the relevant reference_TYPE table name. - String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - StringBuilder createCorrelationTypesTable = new StringBuilder(); - createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); - createCorrelationTypesTable.append("id SERIAL PRIMARY KEY,"); - createCorrelationTypesTable.append("display_name text NOT NULL,"); - createCorrelationTypesTable.append("db_table_name text NOT NULL,"); - createCorrelationTypesTable.append("supported integer NOT NULL,"); - createCorrelationTypesTable.append("enabled integer NOT NULL,"); - createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); - createCorrelationTypesTable.append(")"); - String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); - String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); - String instancesValueIdx = getAddValueIndexTemplate(); - String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); - String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); - // NOTE: the db_info table currenly only has 1 row, so having an index - // provides no benefit. - Connection conn = null; - try { - conn = getEphemeralConnection(false); - if (null == conn) { - return false; - } - Statement stmt = conn.createStatement(); - stmt.execute(createOrganizationsTable.toString()); - stmt.execute(createCasesTable.toString()); - stmt.execute(casesIdx1); - stmt.execute(casesIdx2); - stmt.execute(getCreateDataSourcesTableStatement()); - stmt.execute(getAddDataSourcesNameIndexStatement()); - stmt.execute(getAddDataSourcesObjectIdIndexStatement()); - stmt.execute(createReferenceSetsTable.toString()); - stmt.execute(referenceSetsIdx1); - stmt.execute(createCorrelationTypesTable.toString()); - /* - * Note that the essentially useless id column in the following - * table is required for backwards compatibility. Otherwise, the - * name column could be the primary key. - */ - stmt.execute("CREATE TABLE db_info (id SERIAL, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - // Create a separate instance and reference table for each correlation type - List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); - - String reference_type_dbname; - String instance_type_dbname; - for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { - reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); - instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); - - stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); - - // FUTURE: allow more than the FILES type - if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { - stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); - } - } - - } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS - return false; - } catch (CentralRepoException ex) { - LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS - return false; - } finally { - CentralRepoDbUtil.closeConnection(conn); - } - return true; - } - - /** - * Get the template String for creating a new _instances table in a Postgres - * central repository. %s will exist in the template where the name of the - * new table will be addedd. - * - * @return a String which is a template for cretating a new _instances table - */ - static String getCreateArtifactInstancesTableTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return ("CREATE TABLE IF NOT EXISTS %s (id SERIAL PRIMARY KEY,case_id integer NOT NULL," - + "data_source_id integer NOT NULL,value text NOT NULL,file_path text NOT NULL," - + "known_status integer NOT NULL,comment text,file_obj_id BIGINT," - + "CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path)," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"); - } - - /** - * Get the statement String for creating a new data_sources table in a - * Postgres central repository. - * - * @return a String which is a statement for cretating a new data_sources - * table - */ - static String getCreateDataSourcesTableStatement() { - return "CREATE TABLE IF NOT EXISTS data_sources " - + "(id SERIAL PRIMARY KEY,case_id integer NOT NULL,device_id text NOT NULL," - + "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL," - + "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; - } - - /** - * Get the statement for creating an index on the name column of the - * data_sources table. - * - * @return a String which is a statement for adding an index on the name - * column of the data_sources table. - */ - static String getAddDataSourcesNameIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - } - - /** - * Get the statement for creating an index on the data_sources_object_id - * column of the data_sources table. - * - * @return a String which is a statement for adding an index on the - * data_sources_object_id column of the data_sources table. - */ - static String getAddDataSourcesObjectIdIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - } - - /** - * Get the template for creating an index on the case_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the case_id - * column of a _instances table - */ - static String getAddCaseIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - } - - /** - * Get the template for creating an index on the data_source_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * data_source_id column of a _instances table - */ - static String getAddDataSourceIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - } - - /** - * Get the template for creating an index on the value column of an instance - * table. %s will exist in the template where the name of the new table will - * be addedd. - * - * @return a String which is a template for adding an index to the value - * column of a _instances table - */ - static String getAddValueIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - } - - /** - * Get the template for creating an index on the known_status column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * known_status column of a _instances table - */ - static String getAddKnownStatusIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - } - - /** - * Get the template for creating an index on the file_obj_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * file_obj_id column of a _instances table - */ - static String getAddObjectIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; - } - - public boolean insertDefaultDatabaseContent() { - Connection conn = getEphemeralConnection(false); - if (null == conn) { - return false; - } - - boolean result = CentralRepoDbUtil.insertDefaultCorrelationTypes(conn) && CentralRepoDbUtil.insertDefaultOrganization(conn); - CentralRepoDbUtil.closeConnection(conn); - - return result; - } boolean isChanged() { String hostString = ModuleSettings.getConfigSetting("CentralRepository", "db.postgresql.host"); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java index 1a641c5a28..38191e572e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java @@ -116,6 +116,10 @@ abstract class RdbmsCentralRepo implements CentralRepository { */ protected abstract Connection connect() throws CentralRepoException; + /** + * Get an ephemeral connection. + */ + protected abstract Connection getEphemeralConnection(); /** * Add a new name/value pair in the db_info table. * @@ -3414,22 +3418,23 @@ abstract class RdbmsCentralRepo implements CentralRepository { case POSTGRESQL: addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?) " + getConflictClause(); //NON-NLS - addSsidTableTemplate = PostgresCentralRepoSettings.getCreateArtifactInstancesTableTemplate(); - addCaseIdIndexTemplate = PostgresCentralRepoSettings.getAddCaseIdIndexTemplate(); - addDataSourceIdIndexTemplate = PostgresCentralRepoSettings.getAddDataSourceIdIndexTemplate(); - addValueIndexTemplate = PostgresCentralRepoSettings.getAddValueIndexTemplate(); - addKnownStatusIndexTemplate = PostgresCentralRepoSettings.getAddKnownStatusIndexTemplate(); - addObjectIdIndexTemplate = PostgresCentralRepoSettings.getAddObjectIdIndexTemplate(); + // RAMAN TBD: get these from RdbmsCentralRepoSchemaFactory + addSsidTableTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getCreateArtifactInstancesTableTemplate(); + addCaseIdIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddKnownStatusIndexTemplate(); + addObjectIdIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddObjectIdIndexTemplate(); break; case SQLITE: addAttributeSql = "INSERT OR IGNORE INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; //NON-NLS - addSsidTableTemplate = SqliteCentralRepoSettings.getCreateArtifactInstancesTableTemplate(); - addCaseIdIndexTemplate = SqliteCentralRepoSettings.getAddCaseIdIndexTemplate(); - addDataSourceIdIndexTemplate = SqliteCentralRepoSettings.getAddDataSourceIdIndexTemplate(); - addValueIndexTemplate = SqliteCentralRepoSettings.getAddValueIndexTemplate(); - addKnownStatusIndexTemplate = SqliteCentralRepoSettings.getAddKnownStatusIndexTemplate(); - addObjectIdIndexTemplate = SqliteCentralRepoSettings.getAddObjectIdIndexTemplate(); + addSsidTableTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getCreateArtifactInstancesTableTemplate(); + addCaseIdIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddKnownStatusIndexTemplate(); + addObjectIdIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddObjectIdIndexTemplate(); break; default: throw new CentralRepoException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded.", Bundle.AbstractSqlEamDb_cannotUpgrage_message(selectedPlatform.name())); @@ -3586,8 +3591,8 @@ abstract class RdbmsCentralRepo implements CentralRepository { + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"); - statement.execute(SqliteCentralRepoSettings.getAddDataSourcesNameIndexStatement()); - statement.execute(SqliteCentralRepoSettings.getAddDataSourcesObjectIdIndexStatement()); + statement.execute(RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddDataSourcesNameIndexStatement()); + statement.execute(RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddDataSourcesObjectIdIndexStatement()); statement.execute("INSERT INTO data_sources SELECT * FROM old_data_sources"); statement.execute("DROP TABLE old_data_sources"); break; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java new file mode 100644 index 0000000000..1db99ee3b7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java @@ -0,0 +1,696 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.centralrepository.datamodel; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.logging.Level; +import static org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepo.SOFTWARE_CR_DB_SCHEMA_VERSION; +import org.sleuthkit.autopsy.coreutils.Logger; +//import static org.sleuthkit.autopsy.centralrepository.datamodel.SqliteCentralRepoSettings.getCreateArtifactInstancesTableTemplate; + +/** + * Creates the CR schema and populates it with initial data. + * + */ +public class RdbmsCentralRepoSchemaFactory { + + private final static Logger LOGGER = Logger.getLogger(RdbmsCentralRepoSchemaFactory.class.getName()); + + private static RdbmsCentralRepoSchemaFactory instance; + private final RdbmsCentralRepo rdbmsCentralRepo; + + private final CentralRepoPlatforms selectedPlatform; + + // SQLite pragmas + private final static String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF"; + private final static String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL"; + private final static String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL"; + private final static String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True"; + private final static String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'"; + private final static String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096"; + private final static String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON"; + + /** + * Returns instance of singleton. + * + * @throws CentralRepoException + */ +// public static RdbmsCentralRepoSchemaFactory getInstance() throws CentralRepoException { +// +// if (instance == null) { +// instance = new RdbmsCentralRepoSchemaFactory(); +// } +// +// return instance; +// } + public RdbmsCentralRepoSchemaFactory(CentralRepoPlatforms selectedPlatform) throws CentralRepoException { + //CentralRepoPlatforms selectedPlatform = CentralRepoPlatforms.DISABLED; + //if (CentralRepoDbUtil.allowUseOfCentralRepository()) { + // selectedPlatform = CentralRepoPlatforms.getSelectedPlatform(); + //} + + this.selectedPlatform = selectedPlatform; + switch (selectedPlatform) { + case POSTGRESQL: + rdbmsCentralRepo = PostgresCentralRepo.getInstance(); + break; + case SQLITE: + rdbmsCentralRepo = SqliteCentralRepo.getInstance(); + break; + default: + throw new CentralRepoException("Central Repo platform disabled."); + } + } + + public boolean initializeDatabaseSchema() { + switch (selectedPlatform) { + case POSTGRESQL: + // RAMAN TBD + return PostgresCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo); + //break; + case SQLITE: + return SQLiteCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo); + //break; + default: + return false; + } + } + + // TBD RAMAN - temporary container to store all SQLite methods, till we unify... + public static class SQLiteCRSchemaCreator { + + /** + * Initialize the database schema. + * + * Requires valid connectionPool. + * + * This method is called from within connect(), so we cannot call + * connect() to get a connection. This method is called after + * setupConnectionPool(), so it is safe to assume that a valid + * connectionPool exists. The implementation of connect() is + * synchronized, so we can safely use the connectionPool object + * directly. + */ + public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo) { + // The "id" column is an alias for the built-in 64-bit int "rowid" column. + // It is autoincrementing by default and must be of type "integer primary key". + // We've omitted the autoincrement argument because we are not currently + // using the id value to search for specific rows, so we do not care + // if a rowid is re-used after an existing rows was previously deleted. + StringBuilder createOrganizationsTable = new StringBuilder(); + createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); + createOrganizationsTable.append("id integer primary key autoincrement NOT NULL,"); + createOrganizationsTable.append("org_name text NOT NULL,"); + createOrganizationsTable.append("poc_name text NOT NULL,"); + createOrganizationsTable.append("poc_email text NOT NULL,"); + createOrganizationsTable.append("poc_phone text NOT NULL,"); + createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); + createOrganizationsTable.append(")"); + + // NOTE: The organizations will only have a small number of rows, so + // an index is probably not worthwhile. + StringBuilder createCasesTable = new StringBuilder(); + createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); + createCasesTable.append("id integer primary key autoincrement NOT NULL,"); + createCasesTable.append("case_uid text NOT NULL,"); + createCasesTable.append("org_id integer,"); + createCasesTable.append("case_name text NOT NULL,"); + createCasesTable.append("creation_date text NOT NULL,"); + createCasesTable.append("case_number text,"); + createCasesTable.append("examiner_name text,"); + createCasesTable.append("examiner_email text,"); + createCasesTable.append("examiner_phone text,"); + createCasesTable.append("notes text,"); + createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid) ON CONFLICT IGNORE,"); + createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createCasesTable.append(")"); + + // NOTE: when there are few cases in the cases table, these indices may not be worthwhile + String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; + String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; + + StringBuilder createReferenceSetsTable = new StringBuilder(); + createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); + createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,"); + createReferenceSetsTable.append("org_id integer NOT NULL,"); + createReferenceSetsTable.append("set_name text NOT NULL,"); + createReferenceSetsTable.append("version text NOT NULL,"); + createReferenceSetsTable.append("known_status integer NOT NULL,"); + createReferenceSetsTable.append("read_only boolean NOT NULL,"); + createReferenceSetsTable.append("type integer NOT NULL,"); + createReferenceSetsTable.append("import_date text NOT NULL,"); + createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); + createReferenceSetsTable.append(")"); + + String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; + + // Each "%s" will be replaced with the relevant reference_TYPE table name. + StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); + createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); + createReferenceTypesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); + createReferenceTypesTableTemplate.append("reference_set_id integer,"); + createReferenceTypesTableTemplate.append("value text NOT NULL,"); + createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); + createReferenceTypesTableTemplate.append("comment text,"); + createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value) ON CONFLICT IGNORE,"); + createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createReferenceTypesTableTemplate.append(")"); + + // Each "%s" will be replaced with the relevant reference_TYPE table name. + String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + + StringBuilder createCorrelationTypesTable = new StringBuilder(); + createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); + createCorrelationTypesTable.append("id integer primary key autoincrement NOT NULL,"); + createCorrelationTypesTable.append("display_name text NOT NULL,"); + createCorrelationTypesTable.append("db_table_name text NOT NULL,"); + createCorrelationTypesTable.append("supported integer NOT NULL,"); + createCorrelationTypesTable.append("enabled integer NOT NULL,"); + createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); + createCorrelationTypesTable.append(")"); + + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); + + String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); + String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); + String instancesValueIdx = getAddValueIndexTemplate(); + String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); + String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); + + // NOTE: the db_info table currenly only has 1 row, so having an index + // provides no benefit. + Connection conn = null; + try { + conn = rdbmsCentralRepo.getEphemeralConnection(); + if (null == conn) { + return false; + } + Statement stmt = conn.createStatement(); + stmt.execute(PRAGMA_JOURNAL_WAL); + stmt.execute(PRAGMA_SYNC_OFF); + stmt.execute(PRAGMA_READ_UNCOMMITTED_TRUE); + stmt.execute(PRAGMA_ENCODING_UTF8); + stmt.execute(PRAGMA_PAGE_SIZE_4096); + stmt.execute(PRAGMA_FOREIGN_KEYS_ON); + + stmt.execute(createOrganizationsTable.toString()); + + stmt.execute(createCasesTable.toString()); + stmt.execute(casesIdx1); + stmt.execute(casesIdx2); + + stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); + + stmt.execute(createReferenceSetsTable.toString()); + stmt.execute(referenceSetsIdx1); + + stmt.execute(createCorrelationTypesTable.toString()); + + /* + * Note that the essentially useless id column in the following + * table is required for backwards compatibility. Otherwise, the + * name column could be the primary key. + */ + stmt.execute("CREATE TABLE db_info (id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + + // Create a separate instance and reference table for each artifact type + List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + + String reference_type_dbname; + String instance_type_dbname; + for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { + reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); + instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); + + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); + + // FUTURE: allow more than the FILES type + if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); + } + } + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS + return false; + } catch (CentralRepoException ex) { + LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS + return false; + } finally { + CentralRepoDbUtil.closeConnection(conn); + } + return true; + } + + /** + * Get the template String for creating a new _instances table in a + * Sqlite central repository. %s will exist in the template where the + * name of the new table will be addedd. + * + * @return a String which is a template for cretating a new _instances + * table + */ + static String getCreateArtifactInstancesTableTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE TABLE IF NOT EXISTS %s (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,data_source_id integer NOT NULL,value text NOT NULL," + + "file_path text NOT NULL,known_status integer NOT NULL,comment text,file_obj_id integer," + + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; + } + + /** + * Get the template for creating an index on the case_id column of an + * instance table. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for adding an index to the + * case_id column of a _instances table + */ + static String getAddCaseIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; + } + + /** + * Get the template for creating an index on the data_source_id column + * of an instance table. %s will exist in the template where the name of + * the new table will be addedd. + * + * @return a String which is a template for adding an index to the + * data_source_id column of a _instances table + */ + static String getAddDataSourceIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; + } + + /** + * Get the template for creating an index on the value column of an + * instance table. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for adding an index to the value + * column of a _instances table + */ + static String getAddValueIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + } + + /** + * Get the template for creating an index on the known_status column of + * an instance table. %s will exist in the template where the name of + * the new table will be addedd. + * + * @return a String which is a template for adding an index to the + * known_status column of a _instances table + */ + static String getAddKnownStatusIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + } + + /** + * Get the template for creating an index on the file_obj_id column of + * an instance table. %s will exist in the template where the name of + * the new table will be addedd. + * + * @return a String which is a template for adding an index to the + * file_obj_id column of a _instances table + */ + static String getAddObjectIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; + } + + /** + * Get the statement String for creating a new data_sources table in a + * Sqlite central repository. + * + * @return a String which is a statement for cretating a new + * data_sources table + */ + static String getCreateDataSourcesTableStatement() { + return "CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," + + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; + } + + /** + * Get the statement for creating an index on the name column of the + * data_sources table. + * + * @return a String which is a statement for adding an index on the name + * column of the data_sources table. + */ + static String getAddDataSourcesNameIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; + } + + /** + * Get the statement for creating an index on the data_sources_object_id + * column of the data_sources table. + * + * @return a String which is a statement for adding an index on the + * data_sources_object_id column of the data_sources table. + */ + static String getAddDataSourcesObjectIdIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; + } + + } + + // TBD RAMAN - temporary container to store all Pstgres methods, till we unify... + public static class PostgresCRSchemaCreator { + + /** + * Initialize the database schema. + * + * Requires valid connectionPool. + * + * This method is called from within connect(), so we cannot call + * connect() to get a connection. This method is called after + * setupConnectionPool(), so it is safe to assume that a valid + * connectionPool exists. The implementation of connect() is + * synchronized, so we can safely use the connectionPool object + * directly. + */ + public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo) { + // The "id" column is an alias for the built-in 64-bit int "rowid" column. + // It is autoincrementing by default and must be of type "integer primary key". + // We've omitted the autoincrement argument because we are not currently + // using the id value to search for specific rows, so we do not care + // if a rowid is re-used after an existing rows was previously deleted. + StringBuilder createOrganizationsTable = new StringBuilder(); + createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); + createOrganizationsTable.append("id SERIAL PRIMARY KEY,"); + createOrganizationsTable.append("org_name text NOT NULL,"); + createOrganizationsTable.append("poc_name text NOT NULL,"); + createOrganizationsTable.append("poc_email text NOT NULL,"); + createOrganizationsTable.append("poc_phone text NOT NULL,"); + createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); + createOrganizationsTable.append(")"); + + // NOTE: The organizations will only have a small number of rows, so + // an index is probably not worthwhile. + StringBuilder createCasesTable = new StringBuilder(); + createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); + createCasesTable.append("id SERIAL PRIMARY KEY,"); + createCasesTable.append("case_uid text NOT NULL,"); + createCasesTable.append("org_id integer,"); + createCasesTable.append("case_name text NOT NULL,"); + createCasesTable.append("creation_date text NOT NULL,"); + createCasesTable.append("case_number text,"); + createCasesTable.append("examiner_name text,"); + createCasesTable.append("examiner_email text,"); + createCasesTable.append("examiner_phone text,"); + createCasesTable.append("notes text,"); + createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)"); + createCasesTable.append(")"); + + // NOTE: when there are few cases in the cases table, these indices may not be worthwhile + String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; + String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; + + StringBuilder createReferenceSetsTable = new StringBuilder(); + createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); + createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); + createReferenceSetsTable.append("org_id integer NOT NULL,"); + createReferenceSetsTable.append("set_name text NOT NULL,"); + createReferenceSetsTable.append("version text NOT NULL,"); + createReferenceSetsTable.append("known_status integer NOT NULL,"); + createReferenceSetsTable.append("read_only boolean NOT NULL,"); + createReferenceSetsTable.append("type integer NOT NULL,"); + createReferenceSetsTable.append("import_date text NOT NULL,"); + createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); + createReferenceSetsTable.append(")"); + + String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; + + // Each "%s" will be replaced with the relevant reference_TYPE table name. + StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); + createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); + createReferenceTypesTableTemplate.append("id SERIAL PRIMARY KEY,"); + createReferenceTypesTableTemplate.append("reference_set_id integer,"); + createReferenceTypesTableTemplate.append("value text NOT NULL,"); + createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); + createReferenceTypesTableTemplate.append("comment text,"); + createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE (reference_set_id, value),"); + createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createReferenceTypesTableTemplate.append(")"); + + // Each "%s" will be replaced with the relevant reference_TYPE table name. + String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + + StringBuilder createCorrelationTypesTable = new StringBuilder(); + createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); + createCorrelationTypesTable.append("id SERIAL PRIMARY KEY,"); + createCorrelationTypesTable.append("display_name text NOT NULL,"); + createCorrelationTypesTable.append("db_table_name text NOT NULL,"); + createCorrelationTypesTable.append("supported integer NOT NULL,"); + createCorrelationTypesTable.append("enabled integer NOT NULL,"); + createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); + createCorrelationTypesTable.append(")"); + + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); + + String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); + String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); + String instancesValueIdx = getAddValueIndexTemplate(); + String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); + String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); + + // NOTE: the db_info table currenly only has 1 row, so having an index + // provides no benefit. + Connection conn = null; + try { + conn = rdbmsCentralRepo.getEphemeralConnection(); + if (null == conn) { + return false; + } + Statement stmt = conn.createStatement(); + + stmt.execute(createOrganizationsTable.toString()); + + stmt.execute(createCasesTable.toString()); + stmt.execute(casesIdx1); + stmt.execute(casesIdx2); + + stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); + + stmt.execute(createReferenceSetsTable.toString()); + stmt.execute(referenceSetsIdx1); + + stmt.execute(createCorrelationTypesTable.toString()); + + /* + * Note that the essentially useless id column in the following + * table is required for backwards compatibility. Otherwise, the + * name column could be the primary key. + */ + stmt.execute("CREATE TABLE db_info (id SERIAL, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + + // Create a separate instance and reference table for each correlation type + List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + + String reference_type_dbname; + String instance_type_dbname; + for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { + reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); + instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); + + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); + + // FUTURE: allow more than the FILES type + if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); + } + } + + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS + return false; + } catch (CentralRepoException ex) { + LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS + return false; + } finally { + CentralRepoDbUtil.closeConnection(conn); + } + return true; + } + + /** + * Get the template String for creating a new _instances table in a + * Postgres central repository. %s will exist in the template where the + * name of the new table will be addedd. + * + * @return a String which is a template for cretating a new _instances + * table + */ + static String getCreateArtifactInstancesTableTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return ("CREATE TABLE IF NOT EXISTS %s (id SERIAL PRIMARY KEY,case_id integer NOT NULL," + + "data_source_id integer NOT NULL,value text NOT NULL,file_path text NOT NULL," + + "known_status integer NOT NULL,comment text,file_obj_id BIGINT," + + "CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path)," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"); + } + + /** + * Get the template for creating an index on the case_id column of an + * instance table. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for adding an index to the + * case_id column of a _instances table + */ + static String getAddCaseIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; + } + + /** + * Get the template for creating an index on the data_source_id column + * of an instance table. %s will exist in the template where the name of + * the new table will be addedd. + * + * @return a String which is a template for adding an index to the + * data_source_id column of a _instances table + */ + static String getAddDataSourceIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; + } + + /** + * Get the template for creating an index on the value column of an + * instance table. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for adding an index to the value + * column of a _instances table + */ + static String getAddValueIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + } + + /** + * Get the template for creating an index on the known_status column of + * an instance table. %s will exist in the template where the name of + * the new table will be addedd. + * + * @return a String which is a template for adding an index to the + * known_status column of a _instances table + */ + static String getAddKnownStatusIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + } + + /** + * Get the template for creating an index on the file_obj_id column of + * an instance table. %s will exist in the template where the name of + * the new table will be addedd. + * + * @return a String which is a template for adding an index to the + * file_obj_id column of a _instances table + */ + static String getAddObjectIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; + } + + /** + * Get the statement String for creating a new data_sources table in a + * Postgres central repository. + * + * @return a String which is a statement for cretating a new + * data_sources table + */ + static String getCreateDataSourcesTableStatement() { + return "CREATE TABLE IF NOT EXISTS data_sources " + + "(id SERIAL PRIMARY KEY,case_id integer NOT NULL,device_id text NOT NULL," + + "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL," + + "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; + } + + /** + * Get the statement for creating an index on the name column of the + * data_sources table. + * + * @return a String which is a statement for adding an index on the name + * column of the data_sources table. + */ + static String getAddDataSourcesNameIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; + } + + /** + * Get the statement for creating an index on the data_sources_object_id + * column of the data_sources table. + * + * @return a String which is a statement for adding an index on the + * data_sources_object_id column of the data_sources table. + */ + static String getAddDataSourcesObjectIdIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; + } + + } + + public boolean insertDefaultDatabaseContent() { + Connection conn = rdbmsCentralRepo.getEphemeralConnection(); + if (null == conn) { + return false; + } + + boolean result = CentralRepoDbUtil.insertDefaultCorrelationTypes(conn) && CentralRepoDbUtil.insertDefaultOrganization(conn); + CentralRepoDbUtil.closeConnection(conn); + return result; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java index 17d1393bbf..2c3bf174b6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java @@ -113,6 +113,7 @@ final class SqliteCentralRepo extends RdbmsCentralRepo { @Override public void reset() throws CentralRepoException { try { + // RAMAN TBD: this should be moved to RdbmsCentralRepoSchemaFactory ?? acquireExclusiveLock(); Connection conn = connect(); @@ -144,7 +145,9 @@ final class SqliteCentralRepo extends RdbmsCentralRepo { CentralRepoDbUtil.closeConnection(conn); } - dbSettings.insertDefaultDatabaseContent(); + //RdbmsCentralRepoSchemaFactory.getInstance().insertDefaultDatabaseContent(); + RdbmsCentralRepoSchemaFactory centralRepoSchemaFactory = new RdbmsCentralRepoSchemaFactory(CentralRepoPlatforms.SQLITE); + centralRepoSchemaFactory.insertDefaultDatabaseContent(); } finally { releaseExclusiveLock(); } @@ -226,6 +229,10 @@ final class SqliteCentralRepo extends RdbmsCentralRepo { return ""; } + @Override + protected Connection getEphemeralConnection() { + return this.dbSettings.getEphemeralConnection(); + } /** * Add a new name/value pair in the db_info table. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepoSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepoSettings.java index 3b2a424c4a..25b71e1dcc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepoSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepoSettings.java @@ -25,14 +25,11 @@ import java.nio.file.InvalidPathException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; import java.util.logging.Level; import java.util.regex.Pattern; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import static org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepo.SOFTWARE_CR_DB_SCHEMA_VERSION; /** * Settings for the sqlite implementation of the Central Repository database @@ -48,13 +45,7 @@ public final class SqliteCentralRepoSettings { private final static String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS private final static String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS private final static String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS - private final static String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF"; - private final static String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL"; - private final static String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL"; - private final static String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True"; - private final static String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'"; - private final static String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096"; - private final static String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON"; + private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?"; private String dbName; private String dbDirectory; @@ -182,7 +173,7 @@ public final class SqliteCentralRepoSettings { * * @return Connection or null. */ - private Connection getEphemeralConnection() { + Connection getEphemeralConnection() { if (!dbDirectoryExists()) { return null; } @@ -233,312 +224,6 @@ public final class SqliteCentralRepoSettings { return result; } - /** - * Initialize the database schema. - * - * Requires valid connectionPool. - * - * This method is called from within connect(), so we cannot call connect() - * to get a connection. This method is called after setupConnectionPool(), - * so it is safe to assume that a valid connectionPool exists. The - * implementation of connect() is synchronized, so we can safely use the - * connectionPool object directly. - */ - public boolean initializeDatabaseSchema() { - // The "id" column is an alias for the built-in 64-bit int "rowid" column. - // It is autoincrementing by default and must be of type "integer primary key". - // We've omitted the autoincrement argument because we are not currently - // using the id value to search for specific rows, so we do not care - // if a rowid is re-used after an existing rows was previously deleted. - StringBuilder createOrganizationsTable = new StringBuilder(); - createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); - createOrganizationsTable.append("id integer primary key autoincrement NOT NULL,"); - createOrganizationsTable.append("org_name text NOT NULL,"); - createOrganizationsTable.append("poc_name text NOT NULL,"); - createOrganizationsTable.append("poc_email text NOT NULL,"); - createOrganizationsTable.append("poc_phone text NOT NULL,"); - createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); - createOrganizationsTable.append(")"); - - // NOTE: The organizations will only have a small number of rows, so - // an index is probably not worthwhile. - StringBuilder createCasesTable = new StringBuilder(); - createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); - createCasesTable.append("id integer primary key autoincrement NOT NULL,"); - createCasesTable.append("case_uid text NOT NULL,"); - createCasesTable.append("org_id integer,"); - createCasesTable.append("case_name text NOT NULL,"); - createCasesTable.append("creation_date text NOT NULL,"); - createCasesTable.append("case_number text,"); - createCasesTable.append("examiner_name text,"); - createCasesTable.append("examiner_email text,"); - createCasesTable.append("examiner_phone text,"); - createCasesTable.append("notes text,"); - createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid) ON CONFLICT IGNORE,"); - createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createCasesTable.append(")"); - - // NOTE: when there are few cases in the cases table, these indices may not be worthwhile - String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; - String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - - StringBuilder createReferenceSetsTable = new StringBuilder(); - createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); - createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,"); - createReferenceSetsTable.append("org_id integer NOT NULL,"); - createReferenceSetsTable.append("set_name text NOT NULL,"); - createReferenceSetsTable.append("version text NOT NULL,"); - createReferenceSetsTable.append("known_status integer NOT NULL,"); - createReferenceSetsTable.append("read_only boolean NOT NULL,"); - createReferenceSetsTable.append("type integer NOT NULL,"); - createReferenceSetsTable.append("import_date text NOT NULL,"); - createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); - createReferenceSetsTable.append(")"); - - String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; - - // Each "%s" will be replaced with the relevant reference_TYPE table name. - StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); - createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createReferenceTypesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); - createReferenceTypesTableTemplate.append("reference_set_id integer,"); - createReferenceTypesTableTemplate.append("value text NOT NULL,"); - createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); - createReferenceTypesTableTemplate.append("comment text,"); - createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value) ON CONFLICT IGNORE,"); - createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createReferenceTypesTableTemplate.append(")"); - - // Each "%s" will be replaced with the relevant reference_TYPE table name. - String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - - StringBuilder createCorrelationTypesTable = new StringBuilder(); - createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); - createCorrelationTypesTable.append("id integer primary key autoincrement NOT NULL,"); - createCorrelationTypesTable.append("display_name text NOT NULL,"); - createCorrelationTypesTable.append("db_table_name text NOT NULL,"); - createCorrelationTypesTable.append("supported integer NOT NULL,"); - createCorrelationTypesTable.append("enabled integer NOT NULL,"); - createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); - createCorrelationTypesTable.append(")"); - - String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - - String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); - String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); - String instancesValueIdx = getAddValueIndexTemplate(); - String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); - String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); - - // NOTE: the db_info table currenly only has 1 row, so having an index - // provides no benefit. - Connection conn = null; - try { - conn = getEphemeralConnection(); - if (null == conn) { - return false; - } - Statement stmt = conn.createStatement(); - stmt.execute(PRAGMA_JOURNAL_WAL); - stmt.execute(PRAGMA_SYNC_OFF); - stmt.execute(PRAGMA_READ_UNCOMMITTED_TRUE); - stmt.execute(PRAGMA_ENCODING_UTF8); - stmt.execute(PRAGMA_PAGE_SIZE_4096); - stmt.execute(PRAGMA_FOREIGN_KEYS_ON); - - stmt.execute(createOrganizationsTable.toString()); - - stmt.execute(createCasesTable.toString()); - stmt.execute(casesIdx1); - stmt.execute(casesIdx2); - - stmt.execute(getCreateDataSourcesTableStatement()); - stmt.execute(getAddDataSourcesNameIndexStatement()); - stmt.execute(getAddDataSourcesObjectIdIndexStatement()); - - stmt.execute(createReferenceSetsTable.toString()); - stmt.execute(referenceSetsIdx1); - - stmt.execute(createCorrelationTypesTable.toString()); - - /* - * Note that the essentially useless id column in the following - * table is required for backwards compatibility. Otherwise, the - * name column could be the primary key. - */ - stmt.execute("CREATE TABLE db_info (id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - - // Create a separate instance and reference table for each artifact type - List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); - - String reference_type_dbname; - String instance_type_dbname; - for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { - reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); - instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); - - stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); - - // FUTURE: allow more than the FILES type - if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { - stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); - } - } - } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS - return false; - } catch (CentralRepoException ex) { - LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS - return false; - } finally { - CentralRepoDbUtil.closeConnection(conn); - } - return true; - } - - /** - * Get the template String for creating a new _instances table in a Sqlite - * central repository. %s will exist in the template where the name of the - * new table will be addedd. - * - * @return a String which is a template for cretating a new _instances table - */ - static String getCreateArtifactInstancesTableTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE TABLE IF NOT EXISTS %s (id integer primary key autoincrement NOT NULL," - + "case_id integer NOT NULL,data_source_id integer NOT NULL,value text NOT NULL," - + "file_path text NOT NULL,known_status integer NOT NULL,comment text,file_obj_id integer," - + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; - } - - /** - * Get the statement String for creating a new data_sources table in a - * Sqlite central repository. - * - * @return a String which is a statement for cretating a new data_sources - * table - */ - static String getCreateDataSourcesTableStatement() { - return "CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL," - + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," - + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; - } - - /** - * Get the statement for creating an index on the name column of the - * data_sources table. - * - * @return a String which is a statement for adding an index on the name - * column of the data_sources table. - */ - static String getAddDataSourcesNameIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - } - - /** - * Get the statement for creating an index on the data_sources_object_id - * column of the data_sources table. - * - * @return a String which is a statement for adding an index on the - * data_sources_object_id column of the data_sources table. - */ - static String getAddDataSourcesObjectIdIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - } - - /** - * Get the template for creating an index on the case_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the case_id - * column of a _instances table - */ - static String getAddCaseIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - } - - /** - * Get the template for creating an index on the data_source_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * data_source_id column of a _instances table - */ - static String getAddDataSourceIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - } - - /** - * Get the template for creating an index on the value column of an instance - * table. %s will exist in the template where the name of the new table will - * be addedd. - * - * @return a String which is a template for adding an index to the value - * column of a _instances table - */ - static String getAddValueIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - } - - /** - * Get the template for creating an index on the known_status column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * known_status column of a _instances table - */ - static String getAddKnownStatusIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - } - - /** - * Get the template for creating an index on the file_obj_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * file_obj_id column of a _instances table - */ - static String getAddObjectIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; - } - - public boolean insertDefaultDatabaseContent() { - Connection conn = getEphemeralConnection(); - if (null == conn) { - return false; - } - - boolean result = CentralRepoDbUtil.insertDefaultCorrelationTypes(conn) && CentralRepoDbUtil.insertDefaultOrganization(conn); - CentralRepoDbUtil.closeConnection(conn); - return result; - } - boolean isChanged() { String dbNameString = ModuleSettings.getConfigSetting("CentralRepository", "db.sqlite.dbName"); // NON-NLS String dbDirectoryString = ModuleSettings.getConfigSetting("CentralRepository", "db.sqlite.dbDirectory"); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form index 0f39326bec..27eae7629c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.form @@ -133,7 +133,7 @@ - + @@ -410,4 +410,4 @@ - \ No newline at end of file + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java index 271a8a4f04..cc25b69141 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java @@ -35,6 +35,7 @@ import javax.swing.event.DocumentEvent; import javax.swing.event.DocumentListener; import javax.swing.filechooser.FileFilter; import org.netbeans.spi.options.OptionsPanelController; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.corecomponents.TextPrompt; @@ -45,6 +46,7 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoPlatf import org.sleuthkit.autopsy.centralrepository.datamodel.PostgresCentralRepoSettings; import org.sleuthkit.autopsy.centralrepository.datamodel.SqliteCentralRepoSettings; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; +import org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepoSchemaFactory; /** * Configuration dialog for Central Repository database settings. @@ -447,12 +449,19 @@ public class EamDbSettingsDialog extends JDialog { dbCreated = dbSettingsPostgres.createDatabase(); } if (dbCreated) { - result = dbSettingsPostgres.initializeDatabaseSchema() - && dbSettingsPostgres.insertDefaultDatabaseContent(); + try { + RdbmsCentralRepoSchemaFactory centralRepoSchemaFactory = new RdbmsCentralRepoSchemaFactory(selectedPlatform); + + result = centralRepoSchemaFactory.initializeDatabaseSchema() + && centralRepoSchemaFactory.insertDefaultDatabaseContent(); + } catch (CentralRepoException ex) { + logger.log( Level.SEVERE, "Unable to initialize database schema or insert contents into Postgres central repository.", ex); + } } if (!result) { // Remove the incomplete database if (dbCreated) { + // RAMAN TBD: migrate deleteDatabase() to RdbmsCentralRepoSchemaFactory dbSettingsPostgres.deleteDatabase(); } @@ -469,8 +478,14 @@ public class EamDbSettingsDialog extends JDialog { dbCreated = dbSettingsSqlite.createDbDirectory(); } if (dbCreated) { - result = dbSettingsSqlite.initializeDatabaseSchema() - && dbSettingsSqlite.insertDefaultDatabaseContent(); + try { + RdbmsCentralRepoSchemaFactory centralRepoSchemaFactory = new RdbmsCentralRepoSchemaFactory(selectedPlatform); + result = centralRepoSchemaFactory.initializeDatabaseSchema() + && centralRepoSchemaFactory.insertDefaultDatabaseContent(); + } catch (CentralRepoException ex) { + logger.log( Level.SEVERE, "Unable to initialize database schema or insert contents into SQLite central repository.", ex); + } + } if (!result) { if (dbCreated) { From 0e0a7d3f44fb2a581a065321d3baa7ded4529527 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 7 Feb 2020 14:36:51 -0500 Subject: [PATCH 13/59] removed hard coded font sizes --- .../hashdatabase/HashLookupSettingsPanel.form | 56 +++---------------- .../hashdatabase/HashLookupSettingsPanel.java | 29 +++------- 2 files changed, 18 insertions(+), 67 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.form index 0eea228e69..1630ad7b0e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.form @@ -93,30 +93,22 @@ - - - - - - - - - - + - - - - - - + + + + + + - + + @@ -298,11 +290,6 @@ - - - - - @@ -418,11 +405,6 @@ - - - - - @@ -442,11 +424,6 @@ - - - - - @@ -466,11 +443,6 @@ - - - - - @@ -616,11 +588,6 @@ - - - - - @@ -631,11 +598,6 @@ - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.java index 6c38ea7557..22ff4cc989 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashLookupSettingsPanel.java @@ -706,24 +706,20 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan } }); - informationLabel.setFont(informationLabel.getFont().deriveFont(informationLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); org.openide.awt.Mnemonics.setLocalizedText(informationLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.informationLabel.text")); // NOI18N informationScrollPanel.setVerticalScrollBarPolicy(javax.swing.ScrollPaneConstants.VERTICAL_SCROLLBAR_NEVER); - nameLabel.setFont(nameLabel.getFont().deriveFont(nameLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); org.openide.awt.Mnemonics.setLocalizedText(nameLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.nameLabel.text")); // NOI18N hashDbNameLabel.setFont(hashDbNameLabel.getFont().deriveFont(hashDbNameLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); org.openide.awt.Mnemonics.setLocalizedText(hashDbNameLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.hashDbNameLabel.text")); // NOI18N - typeLabel.setFont(typeLabel.getFont().deriveFont(typeLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); org.openide.awt.Mnemonics.setLocalizedText(typeLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.typeLabel.text")); // NOI18N hashDbTypeLabel.setFont(hashDbTypeLabel.getFont().deriveFont(hashDbTypeLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); org.openide.awt.Mnemonics.setLocalizedText(hashDbTypeLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.hashDbTypeLabel.text")); // NOI18N - locationLabel.setFont(locationLabel.getFont().deriveFont(locationLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); org.openide.awt.Mnemonics.setLocalizedText(locationLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.locationLabel.text")); // NOI18N hashDbLocationLabel.setFont(hashDbLocationLabel.getFont().deriveFont(hashDbLocationLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); @@ -854,7 +850,6 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan } }); - sendIngestMessagesCheckBox.setFont(sendIngestMessagesCheckBox.getFont().deriveFont(sendIngestMessagesCheckBox.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); org.openide.awt.Mnemonics.setLocalizedText(sendIngestMessagesCheckBox, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.sendIngestMessagesCheckBox.text")); // NOI18N sendIngestMessagesCheckBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -862,7 +857,6 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan } }); - ingestWarningLabel.setFont(ingestWarningLabel.getFont().deriveFont(ingestWarningLabel.getFont().getStyle() & ~java.awt.Font.BOLD, 11)); ingestWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/hashdatabase/warning16.png"))); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ingestWarningLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.ingestWarningLabel.text")); // NOI18N @@ -878,23 +872,18 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup() - .addComponent(informationLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE) - .addGap(356, 356, 356)) - .addGroup(jPanel1Layout.createSequentialGroup() - .addComponent(indexButton) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(addHashesToDatabaseButton) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addComponent(informationScrollPanel) .addGroup(jPanel1Layout.createSequentialGroup() .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(informationScrollPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 420, Short.MAX_VALUE) .addGroup(jPanel1Layout.createSequentialGroup() - .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(sendIngestMessagesCheckBox) - .addComponent(ingestWarningLabel)) - .addGap(0, 0, Short.MAX_VALUE))) - .addContainerGap()))) + .addComponent(indexButton) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(addHashesToDatabaseButton)) + .addComponent(sendIngestMessagesCheckBox) + .addComponent(ingestWarningLabel) + .addComponent(informationLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 197, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGap(0, 0, Short.MAX_VALUE))) + .addContainerGap()) .addGroup(jPanel1Layout.createSequentialGroup() .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(hashDatabasesLabel) From 3877804a7c1274d56d55b4aa28e65344c766091f Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Mon, 10 Feb 2020 14:13:27 -0500 Subject: [PATCH 14/59] Interim commit - The schema creation code between SQLite and PostGres has been compared, the differences are reconciled but there is still a fair bit of duplicate code, which should be easy to eliminate now. --- .../datamodel/RdbmsCentralRepo.java | 28 +- .../RdbmsCentralRepoSchemaFactory.java | 494 +++++++++--------- 2 files changed, 249 insertions(+), 273 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java index 38191e572e..335020a5e5 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java @@ -3419,22 +3419,22 @@ abstract class RdbmsCentralRepo implements CentralRepository { addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?) " + getConflictClause(); //NON-NLS // RAMAN TBD: get these from RdbmsCentralRepoSchemaFactory - addSsidTableTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getCreateArtifactInstancesTableTemplate(); - addCaseIdIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddCaseIdIndexTemplate(); - addDataSourceIdIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddDataSourceIdIndexTemplate(); - addValueIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddValueIndexTemplate(); - addKnownStatusIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddKnownStatusIndexTemplate(); - addObjectIdIndexTemplate = RdbmsCentralRepoSchemaFactory.PostgresCRSchemaCreator.getAddObjectIdIndexTemplate(); + addSsidTableTemplate = RdbmsCentralRepoSchemaFactory.getCreateArtifactInstancesTableTemplate(CentralRepoPlatforms.POSTGRESQL); + addCaseIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddKnownStatusIndexTemplate(); + addObjectIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddObjectIdIndexTemplate(); break; case SQLITE: addAttributeSql = "INSERT OR IGNORE INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; //NON-NLS - addSsidTableTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getCreateArtifactInstancesTableTemplate(); - addCaseIdIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddCaseIdIndexTemplate(); - addDataSourceIdIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddDataSourceIdIndexTemplate(); - addValueIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddValueIndexTemplate(); - addKnownStatusIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddKnownStatusIndexTemplate(); - addObjectIdIndexTemplate = RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddObjectIdIndexTemplate(); + addSsidTableTemplate = RdbmsCentralRepoSchemaFactory.getCreateArtifactInstancesTableTemplate(CentralRepoPlatforms.SQLITE); + addCaseIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddKnownStatusIndexTemplate(); + addObjectIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddObjectIdIndexTemplate(); break; default: throw new CentralRepoException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded.", Bundle.AbstractSqlEamDb_cannotUpgrage_message(selectedPlatform.name())); @@ -3591,8 +3591,8 @@ abstract class RdbmsCentralRepo implements CentralRepository { + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"); - statement.execute(RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddDataSourcesNameIndexStatement()); - statement.execute(RdbmsCentralRepoSchemaFactory.SQLiteCRSchemaCreator.getAddDataSourcesObjectIdIndexStatement()); + statement.execute(RdbmsCentralRepoSchemaFactory.getAddDataSourcesNameIndexStatement()); + statement.execute(RdbmsCentralRepoSchemaFactory.getAddDataSourcesObjectIdIndexStatement()); statement.execute("INSERT INTO data_sources SELECT * FROM old_data_sources"); statement.execute("DROP TABLE old_data_sources"); break; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java index 1db99ee3b7..8778656d93 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java @@ -69,13 +69,17 @@ public class RdbmsCentralRepoSchemaFactory { } public boolean initializeDatabaseSchema() { + + // RA TEMP - call the method from platform specific inner class. + // Eventually those two methods need to get unified here + switch (selectedPlatform) { case POSTGRESQL: // RAMAN TBD - return PostgresCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo); + return PostgresCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo, selectedPlatform); //break; case SQLITE: - return SQLiteCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo); + return SQLiteCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo, selectedPlatform); //break; default: return false; @@ -97,7 +101,7 @@ public class RdbmsCentralRepoSchemaFactory { * synchronized, so we can safely use the connectionPool object * directly. */ - public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo) { + public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo, CentralRepoPlatforms selectedPlatform ) { // The "id" column is an alias for the built-in 64-bit int "rowid" column. // It is autoincrementing by default and must be of type "integer primary key". // We've omitted the autoincrement argument because we are not currently @@ -105,7 +109,9 @@ public class RdbmsCentralRepoSchemaFactory { // if a rowid is re-used after an existing rows was previously deleted. StringBuilder createOrganizationsTable = new StringBuilder(); createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); - createOrganizationsTable.append("id integer primary key autoincrement NOT NULL,"); + + createOrganizationsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); + //createOrganizationsTable.append("id integer primary key autoincrement NOT NULL,"); createOrganizationsTable.append("org_name text NOT NULL,"); createOrganizationsTable.append("poc_name text NOT NULL,"); createOrganizationsTable.append("poc_email text NOT NULL,"); @@ -117,7 +123,8 @@ public class RdbmsCentralRepoSchemaFactory { // an index is probably not worthwhile. StringBuilder createCasesTable = new StringBuilder(); createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); - createCasesTable.append("id integer primary key autoincrement NOT NULL,"); + createCasesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); + //createCasesTable.append("id integer primary key autoincrement NOT NULL,"); createCasesTable.append("case_uid text NOT NULL,"); createCasesTable.append("org_id integer,"); createCasesTable.append("case_name text NOT NULL,"); @@ -127,8 +134,8 @@ public class RdbmsCentralRepoSchemaFactory { createCasesTable.append("examiner_email text,"); createCasesTable.append("examiner_phone text,"); createCasesTable.append("notes text,"); - createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid) ON CONFLICT IGNORE,"); - createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid)" ).append(getOnConflictIgnoreClause(selectedPlatform)); createCasesTable.append(")"); // NOTE: when there are few cases in the cases table, these indices may not be worthwhile @@ -137,7 +144,8 @@ public class RdbmsCentralRepoSchemaFactory { StringBuilder createReferenceSetsTable = new StringBuilder(); createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); - createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,"); + createReferenceSetsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); + //createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,"); createReferenceSetsTable.append("org_id integer NOT NULL,"); createReferenceSetsTable.append("set_name text NOT NULL,"); createReferenceSetsTable.append("version text NOT NULL,"); @@ -154,12 +162,13 @@ public class RdbmsCentralRepoSchemaFactory { // Each "%s" will be replaced with the relevant reference_TYPE table name. StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createReferenceTypesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); + createReferenceTypesTableTemplate.append(getNumericPrimaryKeyClause("id", selectedPlatform )); + //createReferenceTypesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); createReferenceTypesTableTemplate.append("reference_set_id integer,"); createReferenceTypesTableTemplate.append("value text NOT NULL,"); createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); createReferenceTypesTableTemplate.append("comment text,"); - createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value) ON CONFLICT IGNORE,"); + createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value)").append(getOnConflictIgnoreClause(selectedPlatform)).append(","); createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); createReferenceTypesTableTemplate.append(")"); @@ -169,7 +178,8 @@ public class RdbmsCentralRepoSchemaFactory { StringBuilder createCorrelationTypesTable = new StringBuilder(); createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); - createCorrelationTypesTable.append("id integer primary key autoincrement NOT NULL,"); + createCorrelationTypesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); + //createCorrelationTypesTable.append("id integer primary key autoincrement NOT NULL,"); createCorrelationTypesTable.append("display_name text NOT NULL,"); createCorrelationTypesTable.append("db_table_name text NOT NULL,"); createCorrelationTypesTable.append("supported integer NOT NULL,"); @@ -177,7 +187,7 @@ public class RdbmsCentralRepoSchemaFactory { createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); createCorrelationTypesTable.append(")"); - String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); @@ -194,6 +204,8 @@ public class RdbmsCentralRepoSchemaFactory { return false; } Statement stmt = conn.createStatement(); + + // RA TBD: this PRAGMA code is SQLIte spcific stmt.execute(PRAGMA_JOURNAL_WAL); stmt.execute(PRAGMA_SYNC_OFF); stmt.execute(PRAGMA_READ_UNCOMMITTED_TRUE); @@ -207,7 +219,7 @@ public class RdbmsCentralRepoSchemaFactory { stmt.execute(casesIdx1); stmt.execute(casesIdx2); - stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); stmt.execute(getAddDataSourcesNameIndexStatement()); stmt.execute(getAddDataSourcesObjectIdIndexStatement()); @@ -221,7 +233,13 @@ public class RdbmsCentralRepoSchemaFactory { * table is required for backwards compatibility. Otherwise, the * name column could be the primary key. */ - stmt.execute("CREATE TABLE db_info (id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)"); + StringBuilder dbInfoTable = new StringBuilder(); + dbInfoTable.append("CREATE TABLE db_info ("); + dbInfoTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + dbInfoTable.append("name TEXT UNIQUE NOT NULL,"); + dbInfoTable.append("value TEXT NOT NULL )"); + + stmt.execute(dbInfoTable.toString()); stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); @@ -262,125 +280,23 @@ public class RdbmsCentralRepoSchemaFactory { return true; } - /** - * Get the template String for creating a new _instances table in a - * Sqlite central repository. %s will exist in the template where the - * name of the new table will be addedd. - * - * @return a String which is a template for cretating a new _instances - * table - */ - static String getCreateArtifactInstancesTableTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE TABLE IF NOT EXISTS %s (id integer primary key autoincrement NOT NULL," - + "case_id integer NOT NULL,data_source_id integer NOT NULL,value text NOT NULL," - + "file_path text NOT NULL,known_status integer NOT NULL,comment text,file_obj_id integer," - + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; - } + - /** - * Get the template for creating an index on the case_id column of an - * instance table. %s will exist in the template where the name of the - * new table will be addedd. - * - * @return a String which is a template for adding an index to the - * case_id column of a _instances table - */ - static String getAddCaseIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - } + - /** - * Get the template for creating an index on the data_source_id column - * of an instance table. %s will exist in the template where the name of - * the new table will be addedd. - * - * @return a String which is a template for adding an index to the - * data_source_id column of a _instances table - */ - static String getAddDataSourceIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - } + - /** - * Get the template for creating an index on the value column of an - * instance table. %s will exist in the template where the name of the - * new table will be addedd. - * - * @return a String which is a template for adding an index to the value - * column of a _instances table - */ - static String getAddValueIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - } + - /** - * Get the template for creating an index on the known_status column of - * an instance table. %s will exist in the template where the name of - * the new table will be addedd. - * - * @return a String which is a template for adding an index to the - * known_status column of a _instances table - */ - static String getAddKnownStatusIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - } + - /** - * Get the template for creating an index on the file_obj_id column of - * an instance table. %s will exist in the template where the name of - * the new table will be addedd. - * - * @return a String which is a template for adding an index to the - * file_obj_id column of a _instances table - */ - static String getAddObjectIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; - } + - /** - * Get the statement String for creating a new data_sources table in a - * Sqlite central repository. - * - * @return a String which is a statement for cretating a new - * data_sources table - */ - static String getCreateDataSourcesTableStatement() { - return "CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL," - + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," - + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; - } + - /** - * Get the statement for creating an index on the name column of the - * data_sources table. - * - * @return a String which is a statement for adding an index on the name - * column of the data_sources table. - */ - static String getAddDataSourcesNameIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - } + - /** - * Get the statement for creating an index on the data_sources_object_id - * column of the data_sources table. - * - * @return a String which is a statement for adding an index on the - * data_sources_object_id column of the data_sources table. - */ - static String getAddDataSourcesObjectIdIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - } + } @@ -399,7 +315,7 @@ public class RdbmsCentralRepoSchemaFactory { * synchronized, so we can safely use the connectionPool object * directly. */ - public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo) { + public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo, CentralRepoPlatforms selectedPlatform) { // The "id" column is an alias for the built-in 64-bit int "rowid" column. // It is autoincrementing by default and must be of type "integer primary key". // We've omitted the autoincrement argument because we are not currently @@ -407,7 +323,8 @@ public class RdbmsCentralRepoSchemaFactory { // if a rowid is re-used after an existing rows was previously deleted. StringBuilder createOrganizationsTable = new StringBuilder(); createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); - createOrganizationsTable.append("id SERIAL PRIMARY KEY,"); + //createOrganizationsTable.append("id SERIAL PRIMARY KEY,"); + createOrganizationsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); createOrganizationsTable.append("org_name text NOT NULL,"); createOrganizationsTable.append("poc_name text NOT NULL,"); createOrganizationsTable.append("poc_email text NOT NULL,"); @@ -419,7 +336,8 @@ public class RdbmsCentralRepoSchemaFactory { // an index is probably not worthwhile. StringBuilder createCasesTable = new StringBuilder(); createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); - createCasesTable.append("id SERIAL PRIMARY KEY,"); + createCasesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); + //createCasesTable.append("id SERIAL PRIMARY KEY,"); createCasesTable.append("case_uid text NOT NULL,"); createCasesTable.append("org_id integer,"); createCasesTable.append("case_name text NOT NULL,"); @@ -430,7 +348,8 @@ public class RdbmsCentralRepoSchemaFactory { createCasesTable.append("examiner_phone text,"); createCasesTable.append("notes text,"); createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)"); + //createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)"); + createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)" ).append(getOnConflictIgnoreClause(selectedPlatform)); createCasesTable.append(")"); // NOTE: when there are few cases in the cases table, these indices may not be worthwhile @@ -439,7 +358,8 @@ public class RdbmsCentralRepoSchemaFactory { StringBuilder createReferenceSetsTable = new StringBuilder(); createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); - createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); + createReferenceSetsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); + //createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); createReferenceSetsTable.append("org_id integer NOT NULL,"); createReferenceSetsTable.append("set_name text NOT NULL,"); createReferenceSetsTable.append("version text NOT NULL,"); @@ -456,12 +376,12 @@ public class RdbmsCentralRepoSchemaFactory { // Each "%s" will be replaced with the relevant reference_TYPE table name. StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createReferenceTypesTableTemplate.append("id SERIAL PRIMARY KEY,"); + createReferenceTypesTableTemplate.append(getNumericPrimaryKeyClause("id", selectedPlatform )); createReferenceTypesTableTemplate.append("reference_set_id integer,"); createReferenceTypesTableTemplate.append("value text NOT NULL,"); createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); createReferenceTypesTableTemplate.append("comment text,"); - createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE (reference_set_id, value),"); + createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value)").append(getOnConflictIgnoreClause(selectedPlatform)).append(","); createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); createReferenceTypesTableTemplate.append(")"); @@ -471,7 +391,7 @@ public class RdbmsCentralRepoSchemaFactory { StringBuilder createCorrelationTypesTable = new StringBuilder(); createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); - createCorrelationTypesTable.append("id SERIAL PRIMARY KEY,"); + createCorrelationTypesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); createCorrelationTypesTable.append("display_name text NOT NULL,"); createCorrelationTypesTable.append("db_table_name text NOT NULL,"); createCorrelationTypesTable.append("supported integer NOT NULL,"); @@ -479,7 +399,7 @@ public class RdbmsCentralRepoSchemaFactory { createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); createCorrelationTypesTable.append(")"); - String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); @@ -503,7 +423,7 @@ public class RdbmsCentralRepoSchemaFactory { stmt.execute(casesIdx1); stmt.execute(casesIdx2); - stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); stmt.execute(getAddDataSourcesNameIndexStatement()); stmt.execute(getAddDataSourcesObjectIdIndexStatement()); @@ -513,11 +433,17 @@ public class RdbmsCentralRepoSchemaFactory { stmt.execute(createCorrelationTypesTable.toString()); /* - * Note that the essentially useless id column in the following - * table is required for backwards compatibility. Otherwise, the - * name column could be the primary key. - */ - stmt.execute("CREATE TABLE db_info (id SERIAL, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)"); + * Note that the essentially useless id column in the following + * table is required for backwards compatibility. Otherwise, the + * name column could be the primary key. + */ + StringBuilder dbInfoTable = new StringBuilder(); + dbInfoTable.append("CREATE TABLE db_info ("); + dbInfoTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + dbInfoTable.append("name TEXT UNIQUE NOT NULL,"); + dbInfoTable.append("value TEXT NOT NULL )"); + + stmt.execute(dbInfoTable.toString()); stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); @@ -559,126 +485,7 @@ public class RdbmsCentralRepoSchemaFactory { return true; } - /** - * Get the template String for creating a new _instances table in a - * Postgres central repository. %s will exist in the template where the - * name of the new table will be addedd. - * - * @return a String which is a template for cretating a new _instances - * table - */ - static String getCreateArtifactInstancesTableTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return ("CREATE TABLE IF NOT EXISTS %s (id SERIAL PRIMARY KEY,case_id integer NOT NULL," - + "data_source_id integer NOT NULL,value text NOT NULL,file_path text NOT NULL," - + "known_status integer NOT NULL,comment text,file_obj_id BIGINT," - + "CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path)," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"); - } - /** - * Get the template for creating an index on the case_id column of an - * instance table. %s will exist in the template where the name of the - * new table will be addedd. - * - * @return a String which is a template for adding an index to the - * case_id column of a _instances table - */ - static String getAddCaseIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - } - - /** - * Get the template for creating an index on the data_source_id column - * of an instance table. %s will exist in the template where the name of - * the new table will be addedd. - * - * @return a String which is a template for adding an index to the - * data_source_id column of a _instances table - */ - static String getAddDataSourceIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - } - - /** - * Get the template for creating an index on the value column of an - * instance table. %s will exist in the template where the name of the - * new table will be addedd. - * - * @return a String which is a template for adding an index to the value - * column of a _instances table - */ - static String getAddValueIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - } - - /** - * Get the template for creating an index on the known_status column of - * an instance table. %s will exist in the template where the name of - * the new table will be addedd. - * - * @return a String which is a template for adding an index to the - * known_status column of a _instances table - */ - static String getAddKnownStatusIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - } - - /** - * Get the template for creating an index on the file_obj_id column of - * an instance table. %s will exist in the template where the name of - * the new table will be addedd. - * - * @return a String which is a template for adding an index to the - * file_obj_id column of a _instances table - */ - static String getAddObjectIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; - } - - /** - * Get the statement String for creating a new data_sources table in a - * Postgres central repository. - * - * @return a String which is a statement for cretating a new - * data_sources table - */ - static String getCreateDataSourcesTableStatement() { - return "CREATE TABLE IF NOT EXISTS data_sources " - + "(id SERIAL PRIMARY KEY,case_id integer NOT NULL,device_id text NOT NULL," - + "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL," - + "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; - } - - /** - * Get the statement for creating an index on the name column of the - * data_sources table. - * - * @return a String which is a statement for adding an index on the name - * column of the data_sources table. - */ - static String getAddDataSourcesNameIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - } - - /** - * Get the statement for creating an index on the data_sources_object_id - * column of the data_sources table. - * - * @return a String which is a statement for adding an index on the - * data_sources_object_id column of the data_sources table. - */ - static String getAddDataSourcesObjectIdIndexStatement() { - return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - } } @@ -693,4 +500,173 @@ public class RdbmsCentralRepoSchemaFactory { return result; } + /** + * Get the template String for creating a new _instances table in a + * Sqlite central repository. %s will exist in the template where the + * name of the new table will be addedd. + * + * @return a String which is a template for cretating a new _instances + * table + */ + static String getCreateArtifactInstancesTableTemplate(CentralRepoPlatforms selectedPlatform) { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE TABLE IF NOT EXISTS %s (" + + getNumericPrimaryKeyClause("id", selectedPlatform ) + + "case_id integer NOT NULL," + + "data_source_id integer NOT NULL," + + "value text NOT NULL," + + "file_path text NOT NULL," + + "known_status integer NOT NULL," + + "comment text," + + "file_obj_id BIGINT," + + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path)" + getOnConflictIgnoreClause(selectedPlatform) + "," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; + } + + + /** + * Get the statement String for creating a new data_sources table in a + * Sqlite central repository. + * + * @return a String which is a statement for creating a new + * data_sources table + */ + static String getCreateDataSourcesTableStatement(CentralRepoPlatforms selectedPlatform) { + return "CREATE TABLE IF NOT EXISTS data_sources (" + + getNumericPrimaryKeyClause("id", selectedPlatform ) + + "case_id integer NOT NULL," + + "device_id text NOT NULL," + + "name text NOT NULL," + + "datasource_obj_id integer," // RAMAN TBD: does integer suffice for PostGres. Old code used integer in some places but used BIGINT in other. + + "md5 text DEFAULT NULL," + + "sha1 text DEFAULT NULL," + + "sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; + } + + /** + * Get the template for creating an index on the case_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be added. + * + * @return a String which is a template for adding an index to the case_id + * column of a _instances table + */ + static String getAddCaseIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; + } + + /** + * Get the template for creating an index on the data_source_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be added. + * + * @return a String which is a template for adding an index to the + * data_source_id column of a _instances table + */ + static String getAddDataSourceIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; + } + + + /** + * Get the template for creating an index on the value column of an instance + * table. %s will exist in the template where the name of the new table will + * be addedd. + * + * @return a String which is a template for adding an index to the value + * column of a _instances table + */ + static String getAddValueIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + } + + /** + * Get the template for creating an index on the known_status column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * known_status column of a _instances table + */ + static String getAddKnownStatusIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + } + + /** + * Get the template for creating an index on the file_obj_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * file_obj_id column of a _instances table + */ + static String getAddObjectIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; + } + + /** + * Get the statement for creating an index on the name column of the + * data_sources table. + * + * @return a String which is a statement for adding an index on the name + * column of the data_sources table. + */ + static String getAddDataSourcesNameIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; + } + + /** + * Get the statement for creating an index on the data_sources_object_id + * column of the data_sources table. + * + * @return a String which is a statement for adding an index on the + * data_sources_object_id column of the data_sources table. + */ + static String getAddDataSourcesObjectIdIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; + } + + /** + * Builds SQL clause for a numeric primary key. Produces correct SQL based + * on the selected CR platform/RDMBS. + * + * @param pkName name of primary key. + * + * @return SQL clause to be used in a Create table statement + */ + // RA TEMP: selectedPlatform is passed as argument, eventually this should be used from the class and this method should not be static + private static String getNumericPrimaryKeyClause(String pkName, CentralRepoPlatforms selectedPlatform) { + switch (selectedPlatform) { + case POSTGRESQL: + return String.format(" %s SERIAL PRIMARY KEY, ", pkName); + case SQLITE: + return String.format(" %s integer primary key autoincrement NOT NULL ,", pkName); + } + return ""; + } + + /** + * Returns ON CONFLICT IGNORE clause for the specified database platform. + * + * + * @return SQL clause. + */ + // RA TEMP: selectedPlatform is passed as argument, eventually this should be used from the class and this method should not be static + private static String getOnConflictIgnoreClause(CentralRepoPlatforms selectedPlatform) { + switch (selectedPlatform) { + case POSTGRESQL: + return ""; + case SQLITE: + return " ON CONFLICT IGNORE "; + } + return ""; + } } From 9e56b863c9d260b02b7525ade6cc088efdef9692 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 11 Feb 2020 09:32:37 -0500 Subject: [PATCH 15/59] Update ExtractRegistry.java Add mpmru parsing, change where artifacts are added in adobemru so it does not get checked every time. --- .../recentactivity/ExtractRegistry.java | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index b21b37ded1..47f70d01dd 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -1172,6 +1172,8 @@ class ExtractRegistry extends Extract { if (line.matches("^adoberdr v.*")) { parseAdobeMRUList(regFileName, regFile, reader); + } else if (line.matches("^mpmru v.*")) { + parseMediaPlayerMRUList(regFileName, regFile, reader); } line = reader.readLine(); } @@ -1235,10 +1237,51 @@ class ExtractRegistry extends Extract { } line = line.trim(); } - if (bbartifacts != null) { - postArtifacts(bbartifacts); + } + if (bbartifacts != null) { + postArtifacts(bbartifacts); + } + } + + /** + * Create recently used artifacts from mpmru records + * + * @param regFileName name of the regripper output file + * + * @param regFile registry file the artifact is associated with + * + * @param reader buffered reader to parse adobemru records + * + * @throws FileNotFound and IOException + */ + private void parseMediaPlayerMRUList(String regFileName, AbstractFile regFile, BufferedReader reader) throws FileNotFoundException, IOException { + List bbartifacts = new ArrayList<>(); + String line = reader.readLine(); + while (!line.contains(SECTION_DIVIDER)) { + line = reader.readLine(); + line = line.trim(); + if (line.contains("LastWrite")) { + line = reader.readLine(); + // Columns are + // FileX -> + while (!line.contains(SECTION_DIVIDER)) { + // Split line on "> " which is the record delimiter between position and file + String tokens[] = line.split("> "); + String fileName = tokens[1]; + Collection attributes = new ArrayList<>(); + attributes.add(new BlackboardAttribute(TSK_PATH, getName(), fileName)); + BlackboardArtifact bba = createArtifactWithAttributes(ARTIFACT_TYPE.TSK_RECENT_OBJECT, regFile, attributes); + if(bba != null) { + bbartifacts.add(bba); + } + line = reader.readLine(); + } + line = line.trim(); } } + if (bbartifacts != null) { + postArtifacts(bbartifacts); + } } /** From ed1443aa6817c0ac2e17f7adcbf95b2f464d7db2 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Tue, 11 Feb 2020 10:54:31 -0500 Subject: [PATCH 16/59] Unified the SQL for schema creation. --- .../datamodel/RdbmsCentralRepo.java | 1 - .../RdbmsCentralRepoSchemaFactory.java | 886 ++++++++---------- .../datamodel/SqliteCentralRepo.java | 1 - 3 files changed, 416 insertions(+), 472 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java index 335020a5e5..ef92944048 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java @@ -3418,7 +3418,6 @@ abstract class RdbmsCentralRepo implements CentralRepository { case POSTGRESQL: addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?) " + getConflictClause(); //NON-NLS - // RAMAN TBD: get these from RdbmsCentralRepoSchemaFactory addSsidTableTemplate = RdbmsCentralRepoSchemaFactory.getCreateArtifactInstancesTableTemplate(CentralRepoPlatforms.POSTGRESQL); addCaseIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = RdbmsCentralRepoSchemaFactory.getAddDataSourceIdIndexTemplate(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java index 8778656d93..6e2e5660eb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Central Repository + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.centralrepository.datamodel; @@ -12,7 +25,6 @@ import java.util.List; import java.util.logging.Level; import static org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepo.SOFTWARE_CR_DB_SCHEMA_VERSION; import org.sleuthkit.autopsy.coreutils.Logger; -//import static org.sleuthkit.autopsy.centralrepository.datamodel.SqliteCentralRepoSettings.getCreateArtifactInstancesTableTemplate; /** * Creates the CR schema and populates it with initial data. @@ -36,26 +48,10 @@ public class RdbmsCentralRepoSchemaFactory { private final static String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096"; private final static String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON"; - /** - * Returns instance of singleton. - * - * @throws CentralRepoException - */ -// public static RdbmsCentralRepoSchemaFactory getInstance() throws CentralRepoException { -// -// if (instance == null) { -// instance = new RdbmsCentralRepoSchemaFactory(); -// } -// -// return instance; -// } - public RdbmsCentralRepoSchemaFactory(CentralRepoPlatforms selectedPlatform) throws CentralRepoException { - //CentralRepoPlatforms selectedPlatform = CentralRepoPlatforms.DISABLED; - //if (CentralRepoDbUtil.allowUseOfCentralRepository()) { - // selectedPlatform = CentralRepoPlatforms.getSelectedPlatform(); - //} + public RdbmsCentralRepoSchemaFactory(CentralRepoPlatforms selectedPlatform) throws CentralRepoException { this.selectedPlatform = selectedPlatform; + switch (selectedPlatform) { case POSTGRESQL: rdbmsCentralRepo = PostgresCentralRepo.getInstance(); @@ -68,427 +64,382 @@ public class RdbmsCentralRepoSchemaFactory { } } + + /** + * Initialize the database schema. + * + * Requires valid connectionPool. + * + * This method is called from within connect(), so we cannot call connect() + * to get a connection. This method is called after setupConnectionPool(), + * so it is safe to assume that a valid connectionPool exists. The + * implementation of connect() is synchronized, so we can safely use the + * connectionPool object directly. + */ public boolean initializeDatabaseSchema() { - - // RA TEMP - call the method from platform specific inner class. - // Eventually those two methods need to get unified here - - switch (selectedPlatform) { - case POSTGRESQL: - // RAMAN TBD - return PostgresCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo, selectedPlatform); - //break; - case SQLITE: - return SQLiteCRSchemaCreator.initializeDatabaseSchema(rdbmsCentralRepo, selectedPlatform); - //break; - default: + // The "id" column is an alias for the built-in 64-bit int "rowid" column. + // It is autoincrementing by default and must be of type "integer primary key". + // We've omitted the autoincrement argument because we are not currently + // using the id value to search for specific rows, so we do not care + // if a rowid is re-used after an existing rows was previously deleted. + StringBuilder createOrganizationsTable = new StringBuilder(); + createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); + + createOrganizationsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + createOrganizationsTable.append("org_name text NOT NULL,"); + createOrganizationsTable.append("poc_name text NOT NULL,"); + createOrganizationsTable.append("poc_email text NOT NULL,"); + createOrganizationsTable.append("poc_phone text NOT NULL,"); + createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); + createOrganizationsTable.append(")"); + + // NOTE: The organizations will only have a small number of rows, so + // an index is probably not worthwhile. + StringBuilder createCasesTable = new StringBuilder(); + createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); + createCasesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + createCasesTable.append("case_uid text NOT NULL,"); + createCasesTable.append("org_id integer,"); + createCasesTable.append("case_name text NOT NULL,"); + createCasesTable.append("creation_date text NOT NULL,"); + createCasesTable.append("case_number text,"); + createCasesTable.append("examiner_name text,"); + createCasesTable.append("examiner_email text,"); + createCasesTable.append("examiner_phone text,"); + createCasesTable.append("notes text,"); + createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid)").append(getOnConflictIgnoreClause(selectedPlatform)); + createCasesTable.append(")"); + + // NOTE: when there are few cases in the cases table, these indices may not be worthwhile + String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; + String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; + + StringBuilder createReferenceSetsTable = new StringBuilder(); + createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); + createReferenceSetsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + createReferenceSetsTable.append("org_id integer NOT NULL,"); + createReferenceSetsTable.append("set_name text NOT NULL,"); + createReferenceSetsTable.append("version text NOT NULL,"); + createReferenceSetsTable.append("known_status integer NOT NULL,"); + createReferenceSetsTable.append("read_only boolean NOT NULL,"); + createReferenceSetsTable.append("type integer NOT NULL,"); + createReferenceSetsTable.append("import_date text NOT NULL,"); + createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); + createReferenceSetsTable.append(")"); + + String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; + + // Each "%s" will be replaced with the relevant reference_TYPE table name. + StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); + createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); + createReferenceTypesTableTemplate.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + createReferenceTypesTableTemplate.append("reference_set_id integer,"); + createReferenceTypesTableTemplate.append("value text NOT NULL,"); + createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); + createReferenceTypesTableTemplate.append("comment text,"); + createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value)").append(getOnConflictIgnoreClause(selectedPlatform)).append(","); + createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createReferenceTypesTableTemplate.append(")"); + + // Each "%s" will be replaced with the relevant reference_TYPE table name. + String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + + StringBuilder createCorrelationTypesTable = new StringBuilder(); + createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); + createCorrelationTypesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + createCorrelationTypesTable.append("display_name text NOT NULL,"); + createCorrelationTypesTable.append("db_table_name text NOT NULL,"); + createCorrelationTypesTable.append("supported integer NOT NULL,"); + createCorrelationTypesTable.append("enabled integer NOT NULL,"); + createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); + createCorrelationTypesTable.append(")"); + + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); + + String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); + String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); + String instancesValueIdx = getAddValueIndexTemplate(); + String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); + String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); + + // NOTE: the db_info table currenly only has 1 row, so having an index + // provides no benefit. + Connection conn = null; + try { + conn = rdbmsCentralRepo.getEphemeralConnection(); + if (null == conn) { return false; - } - } + } + Statement stmt = conn.createStatement(); - // TBD RAMAN - temporary container to store all SQLite methods, till we unify... - public static class SQLiteCRSchemaCreator { - - /** - * Initialize the database schema. - * - * Requires valid connectionPool. - * - * This method is called from within connect(), so we cannot call - * connect() to get a connection. This method is called after - * setupConnectionPool(), so it is safe to assume that a valid - * connectionPool exists. The implementation of connect() is - * synchronized, so we can safely use the connectionPool object - * directly. - */ - public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo, CentralRepoPlatforms selectedPlatform ) { - // The "id" column is an alias for the built-in 64-bit int "rowid" column. - // It is autoincrementing by default and must be of type "integer primary key". - // We've omitted the autoincrement argument because we are not currently - // using the id value to search for specific rows, so we do not care - // if a rowid is re-used after an existing rows was previously deleted. - StringBuilder createOrganizationsTable = new StringBuilder(); - createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); - - createOrganizationsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - //createOrganizationsTable.append("id integer primary key autoincrement NOT NULL,"); - createOrganizationsTable.append("org_name text NOT NULL,"); - createOrganizationsTable.append("poc_name text NOT NULL,"); - createOrganizationsTable.append("poc_email text NOT NULL,"); - createOrganizationsTable.append("poc_phone text NOT NULL,"); - createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); - createOrganizationsTable.append(")"); - - // NOTE: The organizations will only have a small number of rows, so - // an index is probably not worthwhile. - StringBuilder createCasesTable = new StringBuilder(); - createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); - createCasesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - //createCasesTable.append("id integer primary key autoincrement NOT NULL,"); - createCasesTable.append("case_uid text NOT NULL,"); - createCasesTable.append("org_id integer,"); - createCasesTable.append("case_name text NOT NULL,"); - createCasesTable.append("creation_date text NOT NULL,"); - createCasesTable.append("case_number text,"); - createCasesTable.append("examiner_name text,"); - createCasesTable.append("examiner_email text,"); - createCasesTable.append("examiner_phone text,"); - createCasesTable.append("notes text,"); - createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid)" ).append(getOnConflictIgnoreClause(selectedPlatform)); - createCasesTable.append(")"); - - // NOTE: when there are few cases in the cases table, these indices may not be worthwhile - String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; - String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - - StringBuilder createReferenceSetsTable = new StringBuilder(); - createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); - createReferenceSetsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - //createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,"); - createReferenceSetsTable.append("org_id integer NOT NULL,"); - createReferenceSetsTable.append("set_name text NOT NULL,"); - createReferenceSetsTable.append("version text NOT NULL,"); - createReferenceSetsTable.append("known_status integer NOT NULL,"); - createReferenceSetsTable.append("read_only boolean NOT NULL,"); - createReferenceSetsTable.append("type integer NOT NULL,"); - createReferenceSetsTable.append("import_date text NOT NULL,"); - createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); - createReferenceSetsTable.append(")"); - - String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; - - // Each "%s" will be replaced with the relevant reference_TYPE table name. - StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); - createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createReferenceTypesTableTemplate.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - //createReferenceTypesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); - createReferenceTypesTableTemplate.append("reference_set_id integer,"); - createReferenceTypesTableTemplate.append("value text NOT NULL,"); - createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); - createReferenceTypesTableTemplate.append("comment text,"); - createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value)").append(getOnConflictIgnoreClause(selectedPlatform)).append(","); - createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createReferenceTypesTableTemplate.append(")"); - - // Each "%s" will be replaced with the relevant reference_TYPE table name. - String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - - StringBuilder createCorrelationTypesTable = new StringBuilder(); - createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); - createCorrelationTypesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - //createCorrelationTypesTable.append("id integer primary key autoincrement NOT NULL,"); - createCorrelationTypesTable.append("display_name text NOT NULL,"); - createCorrelationTypesTable.append("db_table_name text NOT NULL,"); - createCorrelationTypesTable.append("supported integer NOT NULL,"); - createCorrelationTypesTable.append("enabled integer NOT NULL,"); - createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); - createCorrelationTypesTable.append(")"); - - String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); - - String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); - String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); - String instancesValueIdx = getAddValueIndexTemplate(); - String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); - String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); - - // NOTE: the db_info table currenly only has 1 row, so having an index - // provides no benefit. - Connection conn = null; - try { - conn = rdbmsCentralRepo.getEphemeralConnection(); - if (null == conn) { - return false; - } - Statement stmt = conn.createStatement(); - - // RA TBD: this PRAGMA code is SQLIte spcific + // these setting PRAGMAs are SQLIte spcific + if (selectedPlatform == CentralRepoPlatforms.SQLITE) { stmt.execute(PRAGMA_JOURNAL_WAL); stmt.execute(PRAGMA_SYNC_OFF); stmt.execute(PRAGMA_READ_UNCOMMITTED_TRUE); stmt.execute(PRAGMA_ENCODING_UTF8); stmt.execute(PRAGMA_PAGE_SIZE_4096); stmt.execute(PRAGMA_FOREIGN_KEYS_ON); + } - stmt.execute(createOrganizationsTable.toString()); + stmt.execute(createOrganizationsTable.toString()); - stmt.execute(createCasesTable.toString()); - stmt.execute(casesIdx1); - stmt.execute(casesIdx2); + stmt.execute(createCasesTable.toString()); + stmt.execute(casesIdx1); + stmt.execute(casesIdx2); - stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); - stmt.execute(getAddDataSourcesNameIndexStatement()); - stmt.execute(getAddDataSourcesObjectIdIndexStatement()); + stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); - stmt.execute(createReferenceSetsTable.toString()); - stmt.execute(referenceSetsIdx1); + stmt.execute(createReferenceSetsTable.toString()); + stmt.execute(referenceSetsIdx1); - stmt.execute(createCorrelationTypesTable.toString()); + stmt.execute(createCorrelationTypesTable.toString()); - /* + /* * Note that the essentially useless id column in the following * table is required for backwards compatibility. Otherwise, the * name column could be the primary key. - */ - StringBuilder dbInfoTable = new StringBuilder(); - dbInfoTable.append("CREATE TABLE db_info ("); - dbInfoTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); - dbInfoTable.append("name TEXT UNIQUE NOT NULL,"); - dbInfoTable.append("value TEXT NOT NULL )"); + */ + StringBuilder dbInfoTable = new StringBuilder(); + dbInfoTable.append("CREATE TABLE db_info ("); + dbInfoTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); + dbInfoTable.append("name TEXT UNIQUE NOT NULL,"); + dbInfoTable.append("value TEXT NOT NULL )"); - stmt.execute(dbInfoTable.toString()); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + stmt.execute(dbInfoTable.toString()); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - // Create a separate instance and reference table for each artifact type - List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + // Create a separate instance and reference table for each artifact type + List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); - String reference_type_dbname; - String instance_type_dbname; - for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { - reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); - instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); + String reference_type_dbname; + String instance_type_dbname; + for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { + reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); + instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); - stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); - // FUTURE: allow more than the FILES type - if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { - stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); - } + // FUTURE: allow more than the FILES type + if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); } - } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS - return false; - } catch (CentralRepoException ex) { - LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS - return false; - } finally { - CentralRepoDbUtil.closeConnection(conn); } - return true; + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS + return false; + } catch (CentralRepoException ex) { + LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS + return false; + } finally { + CentralRepoDbUtil.closeConnection(conn); } - - - - - - - - - - - - - - - - - - - + return true; } // TBD RAMAN - temporary container to store all Pstgres methods, till we unify... - public static class PostgresCRSchemaCreator { - - /** - * Initialize the database schema. - * - * Requires valid connectionPool. - * - * This method is called from within connect(), so we cannot call - * connect() to get a connection. This method is called after - * setupConnectionPool(), so it is safe to assume that a valid - * connectionPool exists. The implementation of connect() is - * synchronized, so we can safely use the connectionPool object - * directly. - */ - public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo, CentralRepoPlatforms selectedPlatform) { - // The "id" column is an alias for the built-in 64-bit int "rowid" column. - // It is autoincrementing by default and must be of type "integer primary key". - // We've omitted the autoincrement argument because we are not currently - // using the id value to search for specific rows, so we do not care - // if a rowid is re-used after an existing rows was previously deleted. - StringBuilder createOrganizationsTable = new StringBuilder(); - createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); - //createOrganizationsTable.append("id SERIAL PRIMARY KEY,"); - createOrganizationsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - createOrganizationsTable.append("org_name text NOT NULL,"); - createOrganizationsTable.append("poc_name text NOT NULL,"); - createOrganizationsTable.append("poc_email text NOT NULL,"); - createOrganizationsTable.append("poc_phone text NOT NULL,"); - createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); - createOrganizationsTable.append(")"); - - // NOTE: The organizations will only have a small number of rows, so - // an index is probably not worthwhile. - StringBuilder createCasesTable = new StringBuilder(); - createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); - createCasesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - //createCasesTable.append("id SERIAL PRIMARY KEY,"); - createCasesTable.append("case_uid text NOT NULL,"); - createCasesTable.append("org_id integer,"); - createCasesTable.append("case_name text NOT NULL,"); - createCasesTable.append("creation_date text NOT NULL,"); - createCasesTable.append("case_number text,"); - createCasesTable.append("examiner_name text,"); - createCasesTable.append("examiner_email text,"); - createCasesTable.append("examiner_phone text,"); - createCasesTable.append("notes text,"); - createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - //createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)"); - createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)" ).append(getOnConflictIgnoreClause(selectedPlatform)); - createCasesTable.append(")"); - - // NOTE: when there are few cases in the cases table, these indices may not be worthwhile - String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; - String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - - StringBuilder createReferenceSetsTable = new StringBuilder(); - createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); - createReferenceSetsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - //createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); - createReferenceSetsTable.append("org_id integer NOT NULL,"); - createReferenceSetsTable.append("set_name text NOT NULL,"); - createReferenceSetsTable.append("version text NOT NULL,"); - createReferenceSetsTable.append("known_status integer NOT NULL,"); - createReferenceSetsTable.append("read_only boolean NOT NULL,"); - createReferenceSetsTable.append("type integer NOT NULL,"); - createReferenceSetsTable.append("import_date text NOT NULL,"); - createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); - createReferenceSetsTable.append(")"); - - String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; - - // Each "%s" will be replaced with the relevant reference_TYPE table name. - StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); - createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createReferenceTypesTableTemplate.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - createReferenceTypesTableTemplate.append("reference_set_id integer,"); - createReferenceTypesTableTemplate.append("value text NOT NULL,"); - createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); - createReferenceTypesTableTemplate.append("comment text,"); - createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value)").append(getOnConflictIgnoreClause(selectedPlatform)).append(","); - createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createReferenceTypesTableTemplate.append(")"); - - // Each "%s" will be replaced with the relevant reference_TYPE table name. - String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - - StringBuilder createCorrelationTypesTable = new StringBuilder(); - createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); - createCorrelationTypesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); - createCorrelationTypesTable.append("display_name text NOT NULL,"); - createCorrelationTypesTable.append("db_table_name text NOT NULL,"); - createCorrelationTypesTable.append("supported integer NOT NULL,"); - createCorrelationTypesTable.append("enabled integer NOT NULL,"); - createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); - createCorrelationTypesTable.append(")"); - - String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); - - String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); - String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); - String instancesValueIdx = getAddValueIndexTemplate(); - String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); - String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); - - // NOTE: the db_info table currenly only has 1 row, so having an index - // provides no benefit. - Connection conn = null; - try { - conn = rdbmsCentralRepo.getEphemeralConnection(); - if (null == conn) { - return false; - } - Statement stmt = conn.createStatement(); - - stmt.execute(createOrganizationsTable.toString()); - - stmt.execute(createCasesTable.toString()); - stmt.execute(casesIdx1); - stmt.execute(casesIdx2); - - stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); - stmt.execute(getAddDataSourcesNameIndexStatement()); - stmt.execute(getAddDataSourcesObjectIdIndexStatement()); - - stmt.execute(createReferenceSetsTable.toString()); - stmt.execute(referenceSetsIdx1); - - stmt.execute(createCorrelationTypesTable.toString()); - - /* - * Note that the essentially useless id column in the following - * table is required for backwards compatibility. Otherwise, the - * name column could be the primary key. - */ - StringBuilder dbInfoTable = new StringBuilder(); - dbInfoTable.append("CREATE TABLE db_info ("); - dbInfoTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); - dbInfoTable.append("name TEXT UNIQUE NOT NULL,"); - dbInfoTable.append("value TEXT NOT NULL )"); - - stmt.execute(dbInfoTable.toString()); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - - // Create a separate instance and reference table for each correlation type - List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); - - String reference_type_dbname; - String instance_type_dbname; - for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { - reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); - instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); - - stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); - - // FUTURE: allow more than the FILES type - if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { - stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); - } - } - - } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS - return false; - } catch (CentralRepoException ex) { - LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS - return false; - } finally { - CentralRepoDbUtil.closeConnection(conn); - } - return true; - } - - - - } - +// public static class PostgresCRSchemaCreator { +// +// /** +// * Initialize the database schema. +// * +// * Requires valid connectionPool. +// * +// * This method is called from within connect(), so we cannot call +// * connect() to get a connection. This method is called after +// * setupConnectionPool(), so it is safe to assume that a valid +// * connectionPool exists. The implementation of connect() is +// * synchronized, so we can safely use the connectionPool object +// * directly. +// */ +// public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo, CentralRepoPlatforms selectedPlatform) { +// // The "id" column is an alias for the built-in 64-bit int "rowid" column. +// // It is autoincrementing by default and must be of type "integer primary key". +// // We've omitted the autoincrement argument because we are not currently +// // using the id value to search for specific rows, so we do not care +// // if a rowid is re-used after an existing rows was previously deleted. +// StringBuilder createOrganizationsTable = new StringBuilder(); +// createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); +// //createOrganizationsTable.append("id SERIAL PRIMARY KEY,"); +// createOrganizationsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); +// createOrganizationsTable.append("org_name text NOT NULL,"); +// createOrganizationsTable.append("poc_name text NOT NULL,"); +// createOrganizationsTable.append("poc_email text NOT NULL,"); +// createOrganizationsTable.append("poc_phone text NOT NULL,"); +// createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); +// createOrganizationsTable.append(")"); +// +// // NOTE: The organizations will only have a small number of rows, so +// // an index is probably not worthwhile. +// StringBuilder createCasesTable = new StringBuilder(); +// createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); +// createCasesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); +// //createCasesTable.append("id SERIAL PRIMARY KEY,"); +// createCasesTable.append("case_uid text NOT NULL,"); +// createCasesTable.append("org_id integer,"); +// createCasesTable.append("case_name text NOT NULL,"); +// createCasesTable.append("creation_date text NOT NULL,"); +// createCasesTable.append("case_number text,"); +// createCasesTable.append("examiner_name text,"); +// createCasesTable.append("examiner_email text,"); +// createCasesTable.append("examiner_phone text,"); +// createCasesTable.append("notes text,"); +// createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); +// //createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)"); +// createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)" ).append(getOnConflictIgnoreClause(selectedPlatform)); +// createCasesTable.append(")"); +// +// // NOTE: when there are few cases in the cases table, these indices may not be worthwhile +// String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; +// String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; +// +// StringBuilder createReferenceSetsTable = new StringBuilder(); +// createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); +// createReferenceSetsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); +// //createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); +// createReferenceSetsTable.append("org_id integer NOT NULL,"); +// createReferenceSetsTable.append("set_name text NOT NULL,"); +// createReferenceSetsTable.append("version text NOT NULL,"); +// createReferenceSetsTable.append("known_status integer NOT NULL,"); +// createReferenceSetsTable.append("read_only boolean NOT NULL,"); +// createReferenceSetsTable.append("type integer NOT NULL,"); +// createReferenceSetsTable.append("import_date text NOT NULL,"); +// createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); +// createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); +// createReferenceSetsTable.append(")"); +// +// String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; +// +// // Each "%s" will be replaced with the relevant reference_TYPE table name. +// StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); +// createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); +// createReferenceTypesTableTemplate.append(getNumericPrimaryKeyClause("id", selectedPlatform )); +// createReferenceTypesTableTemplate.append("reference_set_id integer,"); +// createReferenceTypesTableTemplate.append("value text NOT NULL,"); +// createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); +// createReferenceTypesTableTemplate.append("comment text,"); +// createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value)").append(getOnConflictIgnoreClause(selectedPlatform)).append(","); +// createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); +// createReferenceTypesTableTemplate.append(")"); +// +// // Each "%s" will be replaced with the relevant reference_TYPE table name. +// String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; +// String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; +// +// StringBuilder createCorrelationTypesTable = new StringBuilder(); +// createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); +// createCorrelationTypesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); +// createCorrelationTypesTable.append("display_name text NOT NULL,"); +// createCorrelationTypesTable.append("db_table_name text NOT NULL,"); +// createCorrelationTypesTable.append("supported integer NOT NULL,"); +// createCorrelationTypesTable.append("enabled integer NOT NULL,"); +// createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); +// createCorrelationTypesTable.append(")"); +// +// String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); +// +// String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); +// String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); +// String instancesValueIdx = getAddValueIndexTemplate(); +// String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); +// String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); +// +// // NOTE: the db_info table currenly only has 1 row, so having an index +// // provides no benefit. +// Connection conn = null; +// try { +// conn = rdbmsCentralRepo.getEphemeralConnection(); +// if (null == conn) { +// return false; +// } +// Statement stmt = conn.createStatement(); +// +// stmt.execute(createOrganizationsTable.toString()); +// +// stmt.execute(createCasesTable.toString()); +// stmt.execute(casesIdx1); +// stmt.execute(casesIdx2); +// +// stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); +// stmt.execute(getAddDataSourcesNameIndexStatement()); +// stmt.execute(getAddDataSourcesObjectIdIndexStatement()); +// +// stmt.execute(createReferenceSetsTable.toString()); +// stmt.execute(referenceSetsIdx1); +// +// stmt.execute(createCorrelationTypesTable.toString()); +// +// /* +// * Note that the essentially useless id column in the following +// * table is required for backwards compatibility. Otherwise, the +// * name column could be the primary key. +// */ +// StringBuilder dbInfoTable = new StringBuilder(); +// dbInfoTable.append("CREATE TABLE db_info ("); +// dbInfoTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); +// dbInfoTable.append("name TEXT UNIQUE NOT NULL,"); +// dbInfoTable.append("value TEXT NOT NULL )"); +// +// stmt.execute(dbInfoTable.toString()); +// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); +// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); +// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); +// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); +// +// // Create a separate instance and reference table for each correlation type +// List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); +// +// String reference_type_dbname; +// String instance_type_dbname; +// for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { +// reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); +// instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); +// +// stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); +// stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); +// stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); +// stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); +// stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); +// stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); +// +// // FUTURE: allow more than the FILES type +// if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { +// stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); +// stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); +// stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); +// } +// } +// +// } catch (SQLException ex) { +// LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS +// return false; +// } catch (CentralRepoException ex) { +// LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS +// return false; +// } finally { +// CentralRepoDbUtil.closeConnection(conn); +// } +// return true; +// } +// +// +// +// } public boolean insertDefaultDatabaseContent() { Connection conn = rdbmsCentralRepo.getEphemeralConnection(); if (null == conn) { @@ -499,54 +450,52 @@ public class RdbmsCentralRepoSchemaFactory { CentralRepoDbUtil.closeConnection(conn); return result; } - + + /** + * Get the template String for creating a new _instances table in a Sqlite + * central repository. %s will exist in the template where the name of the + * new table will be added. + * + * @return a String which is a template for creating a new _instances table + */ + static String getCreateArtifactInstancesTableTemplate(CentralRepoPlatforms selectedPlatform) { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE TABLE IF NOT EXISTS %s (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "case_id integer NOT NULL," + + "data_source_id integer NOT NULL," + + "value text NOT NULL," + + "file_path text NOT NULL," + + "known_status integer NOT NULL," + + "comment text," + + "file_obj_id BIGINT," + + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path)" + getOnConflictIgnoreClause(selectedPlatform) + "," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; + } + + /** + * Get the statement String for creating a new data_sources table in a + * Sqlite central repository. + * + * @return a String which is a statement for creating a new data_sources + * table + */ + static String getCreateDataSourcesTableStatement(CentralRepoPlatforms selectedPlatform) { + return "CREATE TABLE IF NOT EXISTS data_sources (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "case_id integer NOT NULL," + + "device_id text NOT NULL," + + "name text NOT NULL," + + "datasource_obj_id integer," // RAMAN TBD: does integer suffice for PostGres. Old code used integer in some places but used BIGINT in other. + + "md5 text DEFAULT NULL," + + "sha1 text DEFAULT NULL," + + "sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; + } + /** - * Get the template String for creating a new _instances table in a - * Sqlite central repository. %s will exist in the template where the - * name of the new table will be addedd. - * - * @return a String which is a template for cretating a new _instances - * table - */ - static String getCreateArtifactInstancesTableTemplate(CentralRepoPlatforms selectedPlatform) { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE TABLE IF NOT EXISTS %s (" - + getNumericPrimaryKeyClause("id", selectedPlatform ) - + "case_id integer NOT NULL," - + "data_source_id integer NOT NULL," - + "value text NOT NULL," - + "file_path text NOT NULL," - + "known_status integer NOT NULL," - + "comment text," - + "file_obj_id BIGINT," - + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path)" + getOnConflictIgnoreClause(selectedPlatform) + "," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; - } - - - /** - * Get the statement String for creating a new data_sources table in a - * Sqlite central repository. - * - * @return a String which is a statement for creating a new - * data_sources table - */ - static String getCreateDataSourcesTableStatement(CentralRepoPlatforms selectedPlatform) { - return "CREATE TABLE IF NOT EXISTS data_sources (" - + getNumericPrimaryKeyClause("id", selectedPlatform ) - + "case_id integer NOT NULL," - + "device_id text NOT NULL," - + "name text NOT NULL," - + "datasource_obj_id integer," // RAMAN TBD: does integer suffice for PostGres. Old code used integer in some places but used BIGINT in other. - + "md5 text DEFAULT NULL," - + "sha1 text DEFAULT NULL," - + "sha256 text DEFAULT NULL," - + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; - } - - /** * Get the template for creating an index on the case_id column of an * instance table. %s will exist in the template where the name of the new * table will be added. @@ -558,7 +507,7 @@ public class RdbmsCentralRepoSchemaFactory { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; } - + /** * Get the template for creating an index on the data_source_id column of an * instance table. %s will exist in the template where the name of the new @@ -572,7 +521,6 @@ public class RdbmsCentralRepoSchemaFactory { return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; } - /** * Get the template for creating an index on the value column of an instance * table. %s will exist in the template where the name of the new table will @@ -611,8 +559,8 @@ public class RdbmsCentralRepoSchemaFactory { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; } - - /** + + /** * Get the statement for creating an index on the name column of the * data_sources table. * @@ -633,7 +581,7 @@ public class RdbmsCentralRepoSchemaFactory { static String getAddDataSourcesObjectIdIndexStatement() { return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; } - + /** * Builds SQL clause for a numeric primary key. Produces correct SQL based * on the selected CR platform/RDMBS. @@ -642,7 +590,6 @@ public class RdbmsCentralRepoSchemaFactory { * * @return SQL clause to be used in a Create table statement */ - // RA TEMP: selectedPlatform is passed as argument, eventually this should be used from the class and this method should not be static private static String getNumericPrimaryKeyClause(String pkName, CentralRepoPlatforms selectedPlatform) { switch (selectedPlatform) { case POSTGRESQL: @@ -652,14 +599,13 @@ public class RdbmsCentralRepoSchemaFactory { } return ""; } - - /** + + /** * Returns ON CONFLICT IGNORE clause for the specified database platform. * * * @return SQL clause. */ - // RA TEMP: selectedPlatform is passed as argument, eventually this should be used from the class and this method should not be static private static String getOnConflictIgnoreClause(CentralRepoPlatforms selectedPlatform) { switch (selectedPlatform) { case POSTGRESQL: diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java index 2c3bf174b6..fdf07ef1b7 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java @@ -113,7 +113,6 @@ final class SqliteCentralRepo extends RdbmsCentralRepo { @Override public void reset() throws CentralRepoException { try { - // RAMAN TBD: this should be moved to RdbmsCentralRepoSchemaFactory ?? acquireExclusiveLock(); Connection conn = connect(); From b79b91f502f68f28627679ea47bcdb65b30b333e Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Tue, 11 Feb 2020 11:17:28 -0500 Subject: [PATCH 17/59] Removed commented code. --- .../centralrepository/datamodel/PostgresCentralRepo.java | 3 +-- .../autopsy/centralrepository/datamodel/SqliteCentralRepo.java | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java index 93c82c7bdf..e23ec933bf 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepo.java @@ -131,8 +131,7 @@ final class PostgresCentralRepo extends RdbmsCentralRepo { CentralRepoDbUtil.closeConnection(conn); } - - //dbSettings.insertDefaultDatabaseContent(); + RdbmsCentralRepoSchemaFactory centralRepoSchemaFactory = new RdbmsCentralRepoSchemaFactory(CentralRepoPlatforms.POSTGRESQL); centralRepoSchemaFactory.insertDefaultDatabaseContent(); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java index fdf07ef1b7..fab858f7cd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteCentralRepo.java @@ -144,7 +144,6 @@ final class SqliteCentralRepo extends RdbmsCentralRepo { CentralRepoDbUtil.closeConnection(conn); } - //RdbmsCentralRepoSchemaFactory.getInstance().insertDefaultDatabaseContent(); RdbmsCentralRepoSchemaFactory centralRepoSchemaFactory = new RdbmsCentralRepoSchemaFactory(CentralRepoPlatforms.SQLITE); centralRepoSchemaFactory.insertDefaultDatabaseContent(); } finally { From ac8256b10254ec4072ddccadb51ce8084ec38a11 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Tue, 11 Feb 2020 11:58:09 -0500 Subject: [PATCH 18/59] Removed commented code. --- .../RdbmsCentralRepoSchemaFactory.java | 195 +----------------- 1 file changed, 5 insertions(+), 190 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java index 6e2e5660eb..ad7feed6e7 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoSchemaFactory.java @@ -34,9 +34,7 @@ public class RdbmsCentralRepoSchemaFactory { private final static Logger LOGGER = Logger.getLogger(RdbmsCentralRepoSchemaFactory.class.getName()); - private static RdbmsCentralRepoSchemaFactory instance; private final RdbmsCentralRepo rdbmsCentralRepo; - private final CentralRepoPlatforms selectedPlatform; // SQLite pragmas @@ -252,194 +250,11 @@ public class RdbmsCentralRepoSchemaFactory { return true; } - // TBD RAMAN - temporary container to store all Pstgres methods, till we unify... -// public static class PostgresCRSchemaCreator { -// -// /** -// * Initialize the database schema. -// * -// * Requires valid connectionPool. -// * -// * This method is called from within connect(), so we cannot call -// * connect() to get a connection. This method is called after -// * setupConnectionPool(), so it is safe to assume that a valid -// * connectionPool exists. The implementation of connect() is -// * synchronized, so we can safely use the connectionPool object -// * directly. -// */ -// public static boolean initializeDatabaseSchema(RdbmsCentralRepo rdbmsCentralRepo, CentralRepoPlatforms selectedPlatform) { -// // The "id" column is an alias for the built-in 64-bit int "rowid" column. -// // It is autoincrementing by default and must be of type "integer primary key". -// // We've omitted the autoincrement argument because we are not currently -// // using the id value to search for specific rows, so we do not care -// // if a rowid is re-used after an existing rows was previously deleted. -// StringBuilder createOrganizationsTable = new StringBuilder(); -// createOrganizationsTable.append("CREATE TABLE IF NOT EXISTS organizations ("); -// //createOrganizationsTable.append("id SERIAL PRIMARY KEY,"); -// createOrganizationsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); -// createOrganizationsTable.append("org_name text NOT NULL,"); -// createOrganizationsTable.append("poc_name text NOT NULL,"); -// createOrganizationsTable.append("poc_email text NOT NULL,"); -// createOrganizationsTable.append("poc_phone text NOT NULL,"); -// createOrganizationsTable.append("CONSTRAINT org_name_unique UNIQUE (org_name)"); -// createOrganizationsTable.append(")"); -// -// // NOTE: The organizations will only have a small number of rows, so -// // an index is probably not worthwhile. -// StringBuilder createCasesTable = new StringBuilder(); -// createCasesTable.append("CREATE TABLE IF NOT EXISTS cases ("); -// createCasesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); -// //createCasesTable.append("id SERIAL PRIMARY KEY,"); -// createCasesTable.append("case_uid text NOT NULL,"); -// createCasesTable.append("org_id integer,"); -// createCasesTable.append("case_name text NOT NULL,"); -// createCasesTable.append("creation_date text NOT NULL,"); -// createCasesTable.append("case_number text,"); -// createCasesTable.append("examiner_name text,"); -// createCasesTable.append("examiner_email text,"); -// createCasesTable.append("examiner_phone text,"); -// createCasesTable.append("notes text,"); -// createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); -// //createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)"); -// createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE (case_uid)" ).append(getOnConflictIgnoreClause(selectedPlatform)); -// createCasesTable.append(")"); -// -// // NOTE: when there are few cases in the cases table, these indices may not be worthwhile -// String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; -// String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; -// -// StringBuilder createReferenceSetsTable = new StringBuilder(); -// createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); -// createReferenceSetsTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); -// //createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); -// createReferenceSetsTable.append("org_id integer NOT NULL,"); -// createReferenceSetsTable.append("set_name text NOT NULL,"); -// createReferenceSetsTable.append("version text NOT NULL,"); -// createReferenceSetsTable.append("known_status integer NOT NULL,"); -// createReferenceSetsTable.append("read_only boolean NOT NULL,"); -// createReferenceSetsTable.append("type integer NOT NULL,"); -// createReferenceSetsTable.append("import_date text NOT NULL,"); -// createReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL,"); -// createReferenceSetsTable.append("CONSTRAINT hash_set_unique UNIQUE (set_name, version)"); -// createReferenceSetsTable.append(")"); -// -// String referenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS reference_sets_org_id ON reference_sets (org_id)"; -// -// // Each "%s" will be replaced with the relevant reference_TYPE table name. -// StringBuilder createReferenceTypesTableTemplate = new StringBuilder(); -// createReferenceTypesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); -// createReferenceTypesTableTemplate.append(getNumericPrimaryKeyClause("id", selectedPlatform )); -// createReferenceTypesTableTemplate.append("reference_set_id integer,"); -// createReferenceTypesTableTemplate.append("value text NOT NULL,"); -// createReferenceTypesTableTemplate.append("known_status integer NOT NULL,"); -// createReferenceTypesTableTemplate.append("comment text,"); -// createReferenceTypesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(reference_set_id, value)").append(getOnConflictIgnoreClause(selectedPlatform)).append(","); -// createReferenceTypesTableTemplate.append("foreign key (reference_set_id) references reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL"); -// createReferenceTypesTableTemplate.append(")"); -// -// // Each "%s" will be replaced with the relevant reference_TYPE table name. -// String referenceTypesIdx1 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; -// String referenceTypesIdx2 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; -// -// StringBuilder createCorrelationTypesTable = new StringBuilder(); -// createCorrelationTypesTable.append("CREATE TABLE IF NOT EXISTS correlation_types ("); -// createCorrelationTypesTable.append(getNumericPrimaryKeyClause("id", selectedPlatform )); -// createCorrelationTypesTable.append("display_name text NOT NULL,"); -// createCorrelationTypesTable.append("db_table_name text NOT NULL,"); -// createCorrelationTypesTable.append("supported integer NOT NULL,"); -// createCorrelationTypesTable.append("enabled integer NOT NULL,"); -// createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); -// createCorrelationTypesTable.append(")"); -// -// String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); -// -// String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); -// String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); -// String instancesValueIdx = getAddValueIndexTemplate(); -// String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); -// String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); -// -// // NOTE: the db_info table currenly only has 1 row, so having an index -// // provides no benefit. -// Connection conn = null; -// try { -// conn = rdbmsCentralRepo.getEphemeralConnection(); -// if (null == conn) { -// return false; -// } -// Statement stmt = conn.createStatement(); -// -// stmt.execute(createOrganizationsTable.toString()); -// -// stmt.execute(createCasesTable.toString()); -// stmt.execute(casesIdx1); -// stmt.execute(casesIdx2); -// -// stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); -// stmt.execute(getAddDataSourcesNameIndexStatement()); -// stmt.execute(getAddDataSourcesObjectIdIndexStatement()); -// -// stmt.execute(createReferenceSetsTable.toString()); -// stmt.execute(referenceSetsIdx1); -// -// stmt.execute(createCorrelationTypesTable.toString()); -// -// /* -// * Note that the essentially useless id column in the following -// * table is required for backwards compatibility. Otherwise, the -// * name column could be the primary key. -// */ -// StringBuilder dbInfoTable = new StringBuilder(); -// dbInfoTable.append("CREATE TABLE db_info ("); -// dbInfoTable.append(getNumericPrimaryKeyClause("id", selectedPlatform)); -// dbInfoTable.append("name TEXT UNIQUE NOT NULL,"); -// dbInfoTable.append("value TEXT NOT NULL )"); -// -// stmt.execute(dbInfoTable.toString()); -// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); -// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); -// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); -// stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); -// -// // Create a separate instance and reference table for each correlation type -// List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); -// -// String reference_type_dbname; -// String instance_type_dbname; -// for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { -// reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); -// instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); -// -// stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); -// stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); -// stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); -// stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); -// stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); -// stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); -// -// // FUTURE: allow more than the FILES type -// if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { -// stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); -// stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); -// stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); -// } -// } -// -// } catch (SQLException ex) { -// LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS -// return false; -// } catch (CentralRepoException ex) { -// LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS -// return false; -// } finally { -// CentralRepoDbUtil.closeConnection(conn); -// } -// return true; -// } -// -// -// -// } + /** + * Inserts default data in CR database. + * + * @return True if success, False otherwise. + */ public boolean insertDefaultDatabaseContent() { Connection conn = rdbmsCentralRepo.getEphemeralConnection(); if (null == conn) { From 54f70e91fb0fd054a1775e74d43b89d83bb59b42 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 11 Feb 2020 13:28:28 -0500 Subject: [PATCH 19/59] Added Autopsy Recent Docs Regripper back in Added Autopsy Recent Docs Regripper back. --- .../recentactivity/ExtractRegistry.java | 15 +++++++----- thirdparty/rr/plugins/autopsyrecentdocs.pl | 23 +++++++++++++++---- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index b21b37ded1..b5d51d2153 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -676,12 +676,15 @@ class ExtractRegistry extends Extract { switch (dataType) { case "recentdocs": //NON-NLS - // BlackboardArtifact bbart = tskCase.getContentById(orgId).newArtifact(ARTIFACT_TYPE.TSK_RECENT_OBJECT); - // bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LAST_ACCESSED.getTypeID(), "RecentActivity", dataType, mtime)); - // bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), "RecentActivity", dataType, mtimeItem)); - // bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE.getTypeID(), "RecentActivity", dataType, value)); - // bbart.addAttributes(bbattributes); - // @@@ BC: Why are we ignoring this... + try { + BlackboardArtifact bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_RECENT_OBJECT); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, "RecentActivity", mtime)); +// bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, "RecentActivity", mtimeItem)); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH, "RecentActivity", value)); + bbart.addAttributes(bbattributes); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding recent object artifact to blackboard.", ex); //NON-NLS + } break; case "usb": //NON-NLS try { diff --git a/thirdparty/rr/plugins/autopsyrecentdocs.pl b/thirdparty/rr/plugins/autopsyrecentdocs.pl index 776126175b..4222566fe2 100644 --- a/thirdparty/rr/plugins/autopsyrecentdocs.pl +++ b/thirdparty/rr/plugins/autopsyrecentdocs.pl @@ -45,6 +45,7 @@ sub pluginmain { my $root_key = $reg->get_root_key; my $key_path = "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RecentDocs"; my $key; + my @recentDocs; if ($key = $root_key->get_subkey($key_path)) { #::rptMsg("RecentDocs"); #::rptMsg("**All values printed in MRUList\\MRUListEx order."); @@ -66,7 +67,8 @@ sub pluginmain { my @list = split(/,/,$rdvals{$tag}); foreach my $i (@list) { - ::rptMsg("".$rdvals{$i} . ""); + push(@recentDocs, $rdvals{$i}) + #::rptMsg("".$rdvals{$i} . ""); } } @@ -74,7 +76,7 @@ sub pluginmain { #::rptMsg($key_path." has no values."); #::logMsg("Error: ".$key_path." has no values."); } - ::rptMsg(""); +# ::rptMsg(""); # Get RecentDocs subkeys' values my @subkeys = $key->get_list_of_subkeys(); if (scalar(@subkeys) > 0) { @@ -98,7 +100,8 @@ sub pluginmain { my @list = split(/,/,$rdvals{$tag}); #::rptMsg($tag." = ".$rdvals{$tag}); foreach my $i (@list) { - #::rptMsg("".$rdvals{$i}); + push(@recentDocs, $rdvals{$i}) + #::rptMsg("".$rdvals{$i} . ""); } #::rptMsg(""); @@ -115,8 +118,20 @@ sub pluginmain { else { #::rptMsg($key_path." not found."); } + + my @filtered = uniq(@recentDocs); + my $recentdoc = ""; + foreach $recentdoc (@filtered) { + ::rptMsg("".$recentdoc . ""); + } + ::rptMsg(""); + +} +# Remove duplicate values in Array so documents are only seen once +sub uniq { + my %seen; + grep !$seen{$_}++, @_; } - sub getRDValues { my $key = shift; From ea764e50f29e2356683ddcfaa37e5e40b42d4d65 Mon Sep 17 00:00:00 2001 From: esaunders Date: Tue, 11 Feb 2020 16:43:15 -0500 Subject: [PATCH 20/59] Added multi user support to command line ingest. --- .../commandlineingest/CommandLineCommand.java | 1 + .../CommandLineIngestManager.java | 12 ++++-- .../CommandLineOptionProcessor.java | 38 +++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java index 8888ac72de..29d3a2e9c5 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java @@ -42,6 +42,7 @@ class CommandLineCommand { */ static enum InputType { CASE_NAME, + CASE_TYPE, CASES_BASE_DIR_PATH, CASE_FOLDER_PATH, DATA_SOURCE_PATH, diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java index 4d04e5dbbf..3dfaf53001 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java @@ -38,6 +38,7 @@ import org.netbeans.spi.sendopts.OptionProcessor; import org.openide.LifecycleManager; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.CaseActionException; import org.sleuthkit.autopsy.casemodule.CaseDetails; import org.sleuthkit.autopsy.casemodule.CaseMetadata; @@ -157,7 +158,12 @@ public class CommandLineIngestManager { Map inputs = command.getInputs(); String baseCaseName = inputs.get(CommandLineCommand.InputType.CASE_NAME.name()); String rootOutputDirectory = inputs.get(CommandLineCommand.InputType.CASES_BASE_DIR_PATH.name()); - openCase(baseCaseName, rootOutputDirectory); + CaseType caseType = CaseType.SINGLE_USER_CASE; + String caseTypeString = inputs.get(CommandLineCommand.InputType.CASE_TYPE.name()); + if (caseTypeString != null && caseTypeString.equalsIgnoreCase(CommandLineOptionProcessor.CASETYPE_MULTI)) { + caseType = CaseType.MULTI_USER_CASE; + } + openCase(baseCaseName, rootOutputDirectory, caseType); String outputDirPath = getOutputDirPath(caseForJob); OutputGenerator.saveCreateCaseOutput(caseForJob, outputDirPath, baseCaseName); @@ -340,7 +346,7 @@ public class CommandLineIngestManager { * * @throws CaseActionException */ - private void openCase(String baseCaseName, String rootOutputDirectory) throws CaseActionException { + private void openCase(String baseCaseName, String rootOutputDirectory, CaseType caseType) throws CaseActionException { LOGGER.log(Level.INFO, "Opening case {0} in directory {1}", new Object[]{baseCaseName, rootOutputDirectory}); Path caseDirectoryPath = findCaseDirectory(Paths.get(rootOutputDirectory), baseCaseName); @@ -355,7 +361,7 @@ public class CommandLineIngestManager { Case.createCaseDirectory(caseDirectoryPath.toString(), Case.CaseType.SINGLE_USER_CASE); CaseDetails caseDetails = new CaseDetails(baseCaseName); - Case.createAsCurrentCase(Case.CaseType.SINGLE_USER_CASE, caseDirectoryPath.toString(), caseDetails); + Case.createAsCurrentCase(caseType, caseDirectoryPath.toString(), caseDetails); } caseForJob = Case.getCurrentCase(); diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java index 387d92293e..f23bd5f483 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java @@ -31,6 +31,7 @@ import org.netbeans.spi.sendopts.Env; import org.netbeans.spi.sendopts.Option; import org.netbeans.spi.sendopts.OptionProcessor; import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.featureaccess.FeatureAccessUtils; /** * This class can be used to add command line options to Autopsy @@ -40,6 +41,7 @@ public class CommandLineOptionProcessor extends OptionProcessor { private static final Logger logger = Logger.getLogger(CommandLineOptionProcessor.class.getName()); private final Option caseNameOption = Option.requiredArgument('n', "caseName"); + private final Option caseTypeOption = Option.requiredArgument('t', "caseType"); private final Option caseBaseDirOption = Option.requiredArgument('o', "caseBaseDir"); private final Option createCaseCommandOption = Option.withoutArgument('c', "createCase"); private final Option dataSourcePathOption = Option.requiredArgument('s', "dataSourcePath"); @@ -55,11 +57,15 @@ public class CommandLineOptionProcessor extends OptionProcessor { private final List commands = new ArrayList<>(); + final static String CASETYPE_MULTI = "multi"; + final static String CASETYPE_SINGLE = "single"; + @Override protected Set - \ No newline at end of file + diff --git a/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java index fdddfa2c62..0ac8ecb2ee 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java @@ -25,6 +25,7 @@ import java.awt.Image; import java.awt.GridBagConstraints; import java.awt.Point; import java.awt.event.MouseEvent; +import java.nio.file.Paths; import java.util.concurrent.TimeUnit; import javax.swing.ImageIcon; import javax.swing.JComponent; @@ -100,7 +101,7 @@ final class VideoThumbnailPanel extends javax.swing.JPanel implements ListCellRe imagePanel = new javax.swing.JPanel(); fileSizeLabel = new javax.swing.JLabel(); - countLabel = new javax.swing.JLabel(); + nameLabel = new javax.swing.JLabel(); scoreLabel = new javax.swing.JLabel(); deletedLabel = new javax.swing.JLabel(); @@ -109,14 +110,14 @@ final class VideoThumbnailPanel extends javax.swing.JPanel implements ListCellRe imagePanel.setLayout(new java.awt.GridBagLayout()); scoreLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/red-circle-exclamation.png"))); // NOI18N - scoreLabel.setMaximumSize(new Dimension(DiscoveryUiUtils.getIconSize(),DiscoveryUiUtils.getIconSize())); - scoreLabel.setMinimumSize(new Dimension(DiscoveryUiUtils.getIconSize(),DiscoveryUiUtils.getIconSize())); - scoreLabel.setPreferredSize(new Dimension(DiscoveryUiUtils.getIconSize(),DiscoveryUiUtils.getIconSize())); + scoreLabel.setMaximumSize(new Dimension(org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize(),org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize())); + scoreLabel.setMinimumSize(new Dimension(org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize(),org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize())); + scoreLabel.setPreferredSize(new Dimension(org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize(),org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize())); deletedLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/file-icon-deleted.png"))); // NOI18N - deletedLabel.setMaximumSize(new Dimension(DiscoveryUiUtils.getIconSize(),DiscoveryUiUtils.getIconSize())); - deletedLabel.setMinimumSize(new Dimension(DiscoveryUiUtils.getIconSize(),DiscoveryUiUtils.getIconSize())); - deletedLabel.setPreferredSize(new Dimension(DiscoveryUiUtils.getIconSize(),DiscoveryUiUtils.getIconSize())); + deletedLabel.setMaximumSize(new Dimension(org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize(),org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize())); + deletedLabel.setMinimumSize(new Dimension(org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize(),org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize())); + deletedLabel.setPreferredSize(new Dimension(org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize(),org.sleuthkit.autopsy.filequery.DiscoveryUiUtils.getIconSize())); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -127,52 +128,55 @@ final class VideoThumbnailPanel extends javax.swing.JPanel implements ListCellRe .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(imagePanel, javax.swing.GroupLayout.DEFAULT_SIZE, 776, Short.MAX_VALUE) .addGroup(layout.createSequentialGroup() - .addComponent(fileSizeLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 248, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(countLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 124, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(fileSizeLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(deletedLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(scoreLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addComponent(scoreLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(nameLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addContainerGap()) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addContainerGap() + .addComponent(nameLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 14, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(imagePanel, javax.swing.GroupLayout.PREFERRED_SIZE, 140, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(fileSizeLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 19, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addComponent(deletedLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(scoreLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(countLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 19, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) + .addComponent(deletedLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(fileSizeLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 14, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(scoreLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addContainerGap()) ); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JLabel countLabel; private javax.swing.JLabel deletedLabel; private javax.swing.JLabel fileSizeLabel; private javax.swing.JPanel imagePanel; + private javax.swing.JLabel nameLabel; private javax.swing.JLabel scoreLabel; // End of variables declaration//GEN-END:variables @Messages({ - "# {0} - numberOfInstances", - "VideoThumbnailPanel.countLabel.text=Number of Instances: {0}", + "# {0} - otherInstanceCount", + "VideoThumbnailPanel.nameLabel.more.text= and {0} more", "VideoThumbnailPanel.deleted.text=All instances of file are deleted."}) @Override public Component getListCellRendererComponent(JList list, VideoThumbnailsWrapper value, int index, boolean isSelected, boolean cellHasFocus) { fileSizeLabel.setText(getFileSizeString(value.getResultFile().getFirstInstance().getSize())); - countLabel.setText(Bundle.VideoThumbnailPanel_countLabel_text(value.getResultFile().getAllInstances().size())); + String nameText = Paths.get(value.getResultFile().getFirstInstance().getParentPath(), value.getResultFile().getFirstInstance().getName()).toString(); + if (value.getResultFile().getAllInstances().size() > 1) { + nameText += Bundle.VideoThumbnailPanel_nameLabel_more_text(value.getResultFile().getAllInstances().size() - 1); + } + nameLabel.setText(nameText); addThumbnails(value); imagePanel.setBackground(isSelected ? SELECTION_COLOR : list.getBackground()); DiscoveryUiUtils.setDeletedIcon(value.getResultFile().isDeleted(), deletedLabel); - DiscoveryUiUtils.setScoreIcon(value.getResultFile(), scoreLabel); + DiscoveryUiUtils.setScoreIcon(value.getResultFile(), scoreLabel); setBackground(isSelected ? SELECTION_COLOR : list.getBackground()); return this; } From b5e84692740498ea305f5e8b92ff80650d1d629e Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 20 Feb 2020 19:06:34 -0500 Subject: [PATCH 45/59] 6040 fix missing bundle message and exceptions from other workers --- .../autopsy/filequery/Bundle.properties-MERGED | 15 +++++++-------- .../autopsy/filequery/ImageThumbnailPanel.form | 3 --- .../autopsy/filequery/ImageThumbnailPanel.java | 1 - .../sleuthkit/autopsy/filequery/ResultsPanel.java | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED index de944bf3cf..ea9e846568 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED @@ -14,8 +14,8 @@ DiscoveryUiUtility.megaBytes.text=MB # {1} - units DiscoveryUiUtility.sizeLabel.text=Size: {0} {1} DiscoveryUiUtility.terraBytes.text=TB -# {0} - extension -DocumentPanel.documentType.extension.text=Extension: {0} +# {0} - otherInstanceCount +DocumentPanel.nameLabel.more.text=\ and {0} more DocumentWrapper.previewInitialValue=Preview not generated yet. FileGroup.groupSortingAlgorithm.groupName.text=Group Name FileGroup.groupSortingAlgorithm.groupSize.text=Group Size @@ -175,9 +175,9 @@ FileSorter.SortingMethod.fullPath.displayName=Full Path FileSorter.SortingMethod.keywordlist.displayName=Keyword List Names GroupsListPanel.noResults.message.text=No results were found for the selected filters. GroupsListPanel.noResults.title.text=No results found -# {0} - numberOfInstances -ImageThumbnailPanel.countLabel.text=Number of Instances: {0} ImageThumbnailPanel.isDeleted.text=All instances of file are deleted. +# {0} - otherInstanceCount +ImageThumbnailPanel.nameLabel.more.text=\ and {0} more OpenFileDiscoveryAction.resultsIncomplete.text=Results may be incomplete ResultFile.score.interestingResult.description=At least one instance of the file has an interesting result associated with it. ResultFile.score.notableFile.description=At least one instance of the file was recognized as notable. @@ -210,19 +210,18 @@ FileSearchPanel.stepTwoLabel.text=Step 2: Filter which images to show FileSearchPanel.stepThreeLabel.text=Step 3: Choose display settings DiscoveryTopComponent.stepOneLabel.text=Step 1: Pick File Type DiscoveryTopComponent.documentsButton.text=Documents -DocumentPanel.countLabel.toolTipText= DocumentPanel.fileSizeLabel.toolTipText= -DocumentPanel.documentType.text= DocumentPanel.isDeletedLabel.toolTipText= ImageThumbnailPanel.isDeletedLabel.toolTipText= +ResultsPanel.unableToCreate.text=Unable to create summary. ResultsPanel.viewFileInDir.name=View File in Directory VideoThumbnailPanel.bytes.text=bytes -# {0} - numberOfInstances -VideoThumbnailPanel.countLabel.text=Number of Instances: {0} VideoThumbnailPanel.deleted.text=All instances of file are deleted. VideoThumbnailPanel.gigaBytes.text=GB VideoThumbnailPanel.kiloBytes.text=KB VideoThumbnailPanel.megaBytes.text=MB +# {0} - otherInstanceCount +VideoThumbnailPanel.nameLabel.more.text=\ and {0} more # {0} - fileSize # {1} - units VideoThumbnailPanel.sizeLabel.text=Size: {0} {1} diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.form b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.form index e8b19379c1..b491a2c6de 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.form +++ b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.form @@ -84,9 +84,6 @@ - - - diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java index 0a50a2d41c..b7d0e7340f 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java @@ -70,7 +70,6 @@ public class ImageThumbnailPanel extends javax.swing.JPanel implements ListCellR fileSizeLabel.setToolTipText(""); - org.openide.awt.Mnemonics.setLocalizedText(nameLabel, org.openide.util.NbBundle.getMessage(ImageThumbnailPanel.class, "ImageThumbnailPanel.nameLabel.text")); // NOI18N nameLabel.setToolTipText(""); nameLabel.setMaximumSize(new java.awt.Dimension(159, 12)); nameLabel.setMinimumSize(new java.awt.Dimension(159, 12)); diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java index dced6bf5da..4711303123 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java @@ -702,6 +702,13 @@ public class ResultsPanel extends javax.swing.JPanel { @Override protected void done() { + try { + get(); + } catch (InterruptedException | ExecutionException ex) { + logger.log(Level.WARNING, "Video Worker Exception for file: " + thumbnailWrapper.getResultFile().getFirstInstance().getId(), ex); + } catch (CancellationException ignored) { + //we want to do nothing in response to this since we allow it to be cancelled + } videoThumbnailViewer.repaint(); } } @@ -736,6 +743,13 @@ public class ResultsPanel extends javax.swing.JPanel { @Override protected void done() { + try { + get(); + } catch (InterruptedException | ExecutionException ex) { + logger.log(Level.WARNING, "Image Worker Exception for file: " + thumbnailWrapper.getResultFile().getFirstInstance().getId(), ex); + } catch (CancellationException ignored) { + //we want to do nothing in response to this since we allow it to be cancelled + } imageThumbnailViewer.repaint(); } From 492382ea747f1bae87d2e447f7343268549ae84a Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Fri, 21 Feb 2020 11:22:20 -0500 Subject: [PATCH 46/59] Normalize file path names. normalize path names with unix/linux format instead of windows in Recent Documents by LNK and Recent Doc MRU's --- .../sleuthkit/autopsy/recentactivity/ExtractRegistry.java | 8 ++++++++ .../autopsy/recentactivity/RecentDocumentsByLnk.java | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 2a9b2045f6..2d56c7b6fe 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -66,6 +66,7 @@ import java.util.Set; import java.util.HashSet; import static java.util.Locale.US; import static java.util.TimeZone.getTimeZone; +import org.apache.commons.io.FilenameUtils; import org.openide.util.Lookup; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; @@ -1229,6 +1230,7 @@ class ExtractRegistry extends Extract { if (fileName.charAt(0) == '/') { fileName = fileName.substring(1,fileName.length() - 1); fileName = fileName.replaceFirst("/", ":/"); + fileName = FilenameUtils.normalize(fileName, true); } // Check to see if more then 2 tokens, Date may not be populated, will default to 0 if (tokens.length > 2) { @@ -1285,6 +1287,7 @@ class ExtractRegistry extends Extract { // Split line on "> " which is the record delimiter between position and file String tokens[] = line.split("> "); String fileName = tokens[1]; + fileName = FilenameUtils.normalize(fileName, true); Collection attributes = new ArrayList<>(); attributes.add(new BlackboardAttribute(TSK_PATH, getName(), fileName)); BlackboardArtifact bba = createArtifactWithAttributes(ARTIFACT_TYPE.TSK_RECENT_OBJECT, regFile, attributes); @@ -1326,6 +1329,7 @@ class ExtractRegistry extends Extract { // Split line on "> " which is the record delimiter between position and file String tokens[] = line.split("> "); String fileName = tokens[1]; + fileName = FilenameUtils.normalize(fileName, true); Collection attributes = new ArrayList<>(); attributes.add(new BlackboardAttribute(TSK_PATH, getName(), fileName)); BlackboardArtifact bba = createArtifactWithAttributes(ARTIFACT_TYPE.TSK_RECENT_OBJECT, regFile, attributes); @@ -1368,6 +1372,7 @@ class ExtractRegistry extends Extract { // Split line on "> " which is the record delimiter between position and file String tokens[] = line.split("> "); String fileName = tokens[1]; + fileName = FilenameUtils.normalize(fileName, true); Collection attributes = new ArrayList<>(); attributes.add(new BlackboardAttribute(TSK_PATH, getName(), fileName)); BlackboardArtifact bba = createArtifactWithAttributes(ARTIFACT_TYPE.TSK_RECENT_OBJECT, regFile, attributes); @@ -1405,6 +1410,7 @@ class ExtractRegistry extends Extract { // Columns are // String fileName = line; + fileName = FilenameUtils.normalize(fileName, true); Collection attributes = new ArrayList<>(); attributes.add(new BlackboardAttribute(TSK_PATH, getName(), fileName)); BlackboardArtifact bba = createArtifactWithAttributes(ARTIFACT_TYPE.TSK_RECENT_OBJECT, regFile, attributes); @@ -1448,6 +1454,7 @@ class ExtractRegistry extends Extract { Long docDate = Long.valueOf(tokens[0]); String fileNameTokens[] = tokens[4].split(" - "); String fileName = fileNameTokens[1]; + fileName = FilenameUtils.normalize(fileName, true); Collection attributes = new ArrayList<>(); attributes.add(new BlackboardAttribute(TSK_PATH, getName(), fileName)); attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, getName(), docDate)); @@ -1494,6 +1501,7 @@ class ExtractRegistry extends Extract { String tokens[] = line.split(" : "); fileName = tokens[1]; fileName = fileName.replace("%USERPROFILE%", userProfile); + fileName = FilenameUtils.normalize(fileName, true); // Time in the format of Wed May 31 14:33:03 2017 Z try { String fileUsedTime = tokens[0].replaceAll(" Z",""); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java index 558a29bf8e..7898613373 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java @@ -29,6 +29,7 @@ import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import java.util.Collection; +import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.JLNK; import org.sleuthkit.autopsy.coreutils.JLnkParser; @@ -107,7 +108,7 @@ class RecentDocumentsByLnk extends Extract { } Collection bbattributes = new ArrayList<>(); - String path = lnk.getBestPath(); + String path = FilenameUtils.normalize(lnk.getBestPath(), true); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH, NbBundle.getMessage(this.getClass(), "RecentDocumentsByLnk.parentModuleName.noSpace"), From 7b7291119f9a9d32a7fec57c367bde25e94a33ba Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 21 Feb 2020 11:38:28 -0500 Subject: [PATCH 47/59] 6040 fix paths for name field --- Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java | 2 +- .../org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java | 2 +- .../org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java index 30e4b928db..95a2324ed9 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java @@ -136,7 +136,7 @@ public class DocumentPanel extends javax.swing.JPanel implements ListCellRendere @Override public Component getListCellRendererComponent(JList list, DocumentWrapper value, int index, boolean isSelected, boolean cellHasFocus) { fileSizeLabel.setText(DiscoveryUiUtils.getFileSizeString(value.getResultFile().getFirstInstance().getSize())); - String nameText = Paths.get(value.getResultFile().getFirstInstance().getParentPath(), value.getResultFile().getFirstInstance().getName()).toString(); + String nameText = value.getResultFile().getFirstInstance().getParentPath() + value.getResultFile().getFirstInstance().getName(); if (value.getResultFile().getAllInstances().size() > 1) { nameText += Bundle.DocumentPanel_nameLabel_more_text(value.getResultFile().getAllInstances().size() - 1); } diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java index b7d0e7340f..9ef5572c8d 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java @@ -137,7 +137,7 @@ public class ImageThumbnailPanel extends javax.swing.JPanel implements ListCellR @Override public Component getListCellRendererComponent(JList list, ImageThumbnailWrapper value, int index, boolean isSelected, boolean cellHasFocus) { fileSizeLabel.setText(DiscoveryUiUtils.getFileSizeString(value.getResultFile().getFirstInstance().getSize())); - String nameText = Paths.get(value.getResultFile().getFirstInstance().getParentPath(), value.getResultFile().getFirstInstance().getName()).toString(); + String nameText = value.getResultFile().getFirstInstance().getParentPath() + value.getResultFile().getFirstInstance().getName(); if (value.getResultFile().getAllInstances().size() > 1) { nameText += Bundle.ImageThumbnailPanel_nameLabel_more_text(value.getResultFile().getAllInstances().size() - 1); } diff --git a/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java index 0ac8ecb2ee..4c349316d3 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java @@ -168,7 +168,7 @@ final class VideoThumbnailPanel extends javax.swing.JPanel implements ListCellRe @Override public Component getListCellRendererComponent(JList list, VideoThumbnailsWrapper value, int index, boolean isSelected, boolean cellHasFocus) { fileSizeLabel.setText(getFileSizeString(value.getResultFile().getFirstInstance().getSize())); - String nameText = Paths.get(value.getResultFile().getFirstInstance().getParentPath(), value.getResultFile().getFirstInstance().getName()).toString(); + String nameText = value.getResultFile().getFirstInstance().getParentPath() + value.getResultFile().getFirstInstance().getName(); if (value.getResultFile().getAllInstances().size() > 1) { nameText += Bundle.VideoThumbnailPanel_nameLabel_more_text(value.getResultFile().getAllInstances().size() - 1); } From b611e6b822ae0b4e1f0c13da3bd9d70692a9ff20 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 21 Feb 2020 13:09:37 -0500 Subject: [PATCH 48/59] 6040 fix codacy address comment --- .../org/sleuthkit/autopsy/filequery/DocumentPanel.java | 1 - .../org/sleuthkit/autopsy/filequery/FileSearch.java | 10 +++------- .../autopsy/filequery/ImageThumbnailPanel.java | 1 - .../autopsy/filequery/VideoThumbnailPanel.java | 1 - 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java index 95a2324ed9..849d3843ec 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/DocumentPanel.java @@ -23,7 +23,6 @@ import java.awt.Component; import java.awt.Dimension; import java.awt.Point; import java.awt.event.MouseEvent; -import java.nio.file.Paths; import javax.swing.JComponent; import javax.swing.JList; import javax.swing.ListCellRenderer; diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java index 1f6e65bba9..ee5c36c8e3 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java @@ -264,11 +264,7 @@ class FileSearch { if (localSummarizer == null) { synchronized (searchCache) { if (localSummarizer == null) { - try { - localSummarizer = getLocalSummarizer(); - } catch (IOException ignored) { - //no summarizers being present is not unexpected - } + localSummarizer = getLocalSummarizer(); } } } @@ -313,7 +309,7 @@ class FileSearch { * * @throws IOException */ - private static TextSummarizer getLocalSummarizer() throws IOException { + private static TextSummarizer getLocalSummarizer() { Collection summarizers = Lookup.getDefault().lookupAll(TextSummarizer.class ); @@ -321,7 +317,7 @@ class FileSearch { summarizerToUse = summarizers.iterator().next(); return summarizerToUse; } - throw new IOException("No summarizers found"); + return null; } /** diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java index 9ef5572c8d..78654296cb 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ImageThumbnailPanel.java @@ -23,7 +23,6 @@ import java.awt.Component; import java.awt.Dimension; import java.awt.Point; import java.awt.event.MouseEvent; -import java.nio.file.Paths; import javax.swing.ImageIcon; import javax.swing.JComponent; import javax.swing.JList; diff --git a/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java index 4c349316d3..cff5502e73 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/VideoThumbnailPanel.java @@ -25,7 +25,6 @@ import java.awt.Image; import java.awt.GridBagConstraints; import java.awt.Point; import java.awt.event.MouseEvent; -import java.nio.file.Paths; import java.util.concurrent.TimeUnit; import javax.swing.ImageIcon; import javax.swing.JComponent; From 7de33d4b35ae2c9c1c307ad293707cdf8447a73e Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Mon, 24 Feb 2020 13:40:09 -0500 Subject: [PATCH 49/59] 6014: Update central repository database creation to make personas tables. --- .../datamodel/CentralRepository.java | 11 +- .../CorrelationAttributeInstance.java | 42 ++- .../centralrepository/datamodel/Persona.java | 87 +++++ .../datamodel/RdbmsCentralRepo.java | 74 ++++ .../datamodel/RdbmsCentralRepoFactory.java | 340 +++++++++++++++++- .../optionspanel/EamDbSettingsDialog.java | 1 - .../CommonAttributeCaseSearchResults.java | 5 +- .../CommonAttributeCountSearchResults.java | 7 +- .../CommonAttributePanel.java | 6 +- .../InterCasePanel.java | 5 +- .../CorrelationCaseChildNodeFactory.java | 4 +- .../InterCaseTestUtils.java | 13 +- 12 files changed, 554 insertions(+), 41 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/Persona.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java index 3e7ac158a9..e357d529e5 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java @@ -802,5 +802,14 @@ public interface CentralRepository { * * @throws CentralRepoException */ - public void processSelectClause(String selectClause, InstanceTableCallback instanceTableCallback) throws CentralRepoException; + public void processSelectClause(String selectClause, InstanceTableCallback instanceTableCallback) throws CentralRepoException; + + + /** + * Returns list of all correlation types. + * + * @return list of Correlation types + * @throws CentralRepoException + */ + public List getCorrelationTypes() throws CentralRepoException; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index f13b27787d..a8974f8e5a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2018 Basis Technology Corp. + * Copyright 2015-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,6 +24,7 @@ import java.util.List; import java.util.Objects; import java.util.regex.Pattern; import org.openide.util.NbBundle.Messages; +import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.TskData; /** @@ -220,6 +221,9 @@ public class CorrelationAttributeInstance implements Serializable { public static final int IMEI_TYPE_ID = 7; public static final int IMSI_TYPE_ID = 8; public static final int ICCID_TYPE_ID = 9; + + // An offset to assign Ids for additional correlation types. + public static final int ADDITIONAL_TYPES_BASE_ID = 1000; /** * Load the default correlation types @@ -238,18 +242,30 @@ public class CorrelationAttributeInstance implements Serializable { "CorrelationType.IMSI.displayName=IMSI Number", "CorrelationType.ICCID.displayName=ICCID Number"}) public static List getDefaultCorrelationTypes() throws CentralRepoException { - List DEFAULT_CORRELATION_TYPES = new ArrayList<>(); - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(DOMAIN_TYPE_ID, Bundle.CorrelationType_DOMAIN_displayName(), "domain", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(EMAIL_TYPE_ID, Bundle.CorrelationType_EMAIL_displayName(), "email_address", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(SSID_TYPE_ID, Bundle.CorrelationType_SSID_displayName(), "wireless_networks", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(MAC_TYPE_ID, Bundle.CorrelationType_MAC_displayName(), "mac_address", true, true)); //NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(IMEI_TYPE_ID, Bundle.CorrelationType_IMEI_displayName(), "imei_number", true, true)); //NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(IMSI_TYPE_ID, Bundle.CorrelationType_IMSI_displayName(), "imsi_number", true, true)); //NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(ICCID_TYPE_ID, Bundle.CorrelationType_ICCID_displayName(), "iccid_number", true, true)); //NON-NLS - return DEFAULT_CORRELATION_TYPES; + List defaultCorrelationTypes = new ArrayList<>(); + + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(DOMAIN_TYPE_ID, Bundle.CorrelationType_DOMAIN_displayName(), "domain", true, true)); // NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(EMAIL_TYPE_ID, Bundle.CorrelationType_EMAIL_displayName(), "email_address", true, true)); // NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, true)); // NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, true)); // NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(SSID_TYPE_ID, Bundle.CorrelationType_SSID_displayName(), "wireless_networks", true, true)); // NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(MAC_TYPE_ID, Bundle.CorrelationType_MAC_displayName(), "mac_address", true, true)); //NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(IMEI_TYPE_ID, Bundle.CorrelationType_IMEI_displayName(), "imei_number", true, true)); //NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(IMSI_TYPE_ID, Bundle.CorrelationType_IMSI_displayName(), "imsi_number", true, true)); //NON-NLS + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(ICCID_TYPE_ID, Bundle.CorrelationType_ICCID_displayName(), "iccid_number", true, true)); //NON-NLS + + // Create Correlation Types for Accounts. + int correlationTypeId = ADDITIONAL_TYPES_BASE_ID; + for (Account.Type type : Account.Type.PREDEFINED_ACCOUNT_TYPES) { + // Skip Phone and Email accounts as there are already Correlation types defined for those. + if (type != Account.Type.EMAIL && type != Account.Type.PHONE) { + defaultCorrelationTypes.add(new CorrelationAttributeInstance.Type(correlationTypeId, type.getDisplayName(), type.getTypeName().toLowerCase(), true, true)); //NON-NLS + correlationTypeId++; + } + } + + return defaultCorrelationTypes; } /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/Persona.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/Persona.java new file mode 100644 index 0000000000..5fc458353b --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/Persona.java @@ -0,0 +1,87 @@ +/* + * Central Repository + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.datamodel; + +/** + * This class abstracts a persona. + * + * An examiner may create a persona from an account. + * + */ +class Persona { + + /** + * Defines level of confidence in assigning a persona to an account. + */ + public enum Confidence { + UNKNOWN(1, "Unknown"), + LOW(2, "Low confidence"), + MEDIUM(3, "Medium confidence"), + HIGH(4, "High confidence"), + DERIVED(5, "Derived directly"); + + private final String name; + private final int level_id; + + Confidence(int level, String name) { + this.name = name; + this.level_id = level; + + } + + @Override + public String toString() { + return name; + } + + public int getLevel() { + return this.level_id; + } + } + + /** + * Defines status of a persona. + */ + public enum PersonaStatus { + + UNKNOWN(1, "Unknown"), + ACTIVE(2, "Active"), + MERGED(3, "Merged"), + SPLIT(4, "Split"), + DELETED(5, "Deleted"); + + private final String description; + private final int status_id; + + PersonaStatus(int status, String description) { + this.status_id = status; + this.description = description; + } + + @Override + public String toString() { + return description; + } + + public int getStatus() { + return this.status_id; + } + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java index 513fd3f944..fedf0b75c9 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java @@ -1373,6 +1373,9 @@ abstract class RdbmsCentralRepo implements CentralRepository { } synchronized (bulkArtifacts) { + if (bulkArtifacts.get(CentralRepoDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType())) == null) { + bulkArtifacts.put(CentralRepoDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()), new ArrayList<>()); + } bulkArtifacts.get(CentralRepoDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType())).add(eamArtifact); bulkArtifactsCount++; @@ -2845,6 +2848,7 @@ abstract class RdbmsCentralRepo implements CentralRepository { typeId = newCorrelationTypeKnownId(newType); } + typeCache.put(newType.getId(), newType); return typeId; } @@ -3105,6 +3109,45 @@ abstract class RdbmsCentralRepo implements CentralRepository { } } + /** + * Returns a list of all correlation types. It uses the cache to build the + * list. If the cache is empty, it reads from the database and loads up the + * cache. + * + * @return List of correlation types. + * @throws CentralRepoException + */ + @Override + public List getCorrelationTypes() throws CentralRepoException { + + if (typeCache.size() == 0) { + getCorrelationTypesFromCr(); + } + + return new ArrayList<>(typeCache.asMap().values()); + } + + /** + * Gets a Correlation type with the specified name. + * + * @param correlationtypeName Correlation type name + * @return Correlation type matching the given name, null if none matches. + * + * @throws CentralRepoException + */ + public CorrelationAttributeInstance.Type getCorrelationTypeByName(String correlationtypeName) throws CentralRepoException { + List correlationTypesList = getCorrelationTypes(); + + CorrelationAttributeInstance.Type correlationType + = correlationTypesList.stream() + .filter(x -> correlationtypeName.equalsIgnoreCase(x.getDisplayName())) + .findAny() + .orElse(null); + + return null; + } + + /** * Get the EamArtifact.Type that has the given Type.Id from the central repo * @@ -3142,6 +3185,37 @@ abstract class RdbmsCentralRepo implements CentralRepository { } } + /** + * Reads the correlation types from the database and loads them up in the cache. + * + * @throws CentralRepoException If there is an error. + */ + private void getCorrelationTypesFromCr() throws CentralRepoException { + + // clear out the cache + typeCache.invalidateAll(); + + Connection conn = connect(); + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + + String sql = "SELECT * FROM correlation_types"; + try { + preparedStatement = conn.prepareStatement(sql); + resultSet = preparedStatement.executeQuery(); + while (resultSet.next()) { + CorrelationAttributeInstance.Type aType = getCorrelationTypeFromResultSet(resultSet); + typeCache.put(aType.getId(), aType); + } + } catch (SQLException ex) { + throw new CentralRepoException("Error getting correlation types.", ex); // NON-NLS + } finally { + CentralRepoDbUtil.closeStatement(preparedStatement); + CentralRepoDbUtil.closeResultSet(resultSet); + CentralRepoDbUtil.closeConnection(conn); + } + } + /** * Convert a ResultSet to a EamCase object * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java index 33b6009dea..3ef09b263c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java @@ -19,12 +19,17 @@ package org.sleuthkit.autopsy.centralrepository.datamodel; import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.logging.Level; +import org.sleuthkit.autopsy.centralrepository.datamodel.Persona.Confidence; +import org.sleuthkit.autopsy.centralrepository.datamodel.Persona.PersonaStatus; import static org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepo.SOFTWARE_CR_DB_SCHEMA_VERSION; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Account; /** * Creates the CR schema and populates it with initial data. @@ -132,11 +137,11 @@ public class RdbmsCentralRepoFactory { stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); // Create a separate instance and reference table for each artifact type - List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + List defaultCorrelationTypes = CorrelationAttributeInstance.getDefaultCorrelationTypes(); String reference_type_dbname; String instance_type_dbname; - for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { + for (CorrelationAttributeInstance.Type type : defaultCorrelationTypes) { reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); @@ -154,6 +159,8 @@ public class RdbmsCentralRepoFactory { stmt.execute(String.format(getReferenceTypeValueKnownstatusIndexTemplate(), reference_type_dbname, reference_type_dbname)); } } + + createPersonaTables(stmt); } catch (SQLException ex) { LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS return false; @@ -184,7 +191,11 @@ public class RdbmsCentralRepoFactory { return false; } - boolean result = CentralRepoDbUtil.insertDefaultCorrelationTypes(conn) && CentralRepoDbUtil.insertDefaultOrganization(conn); + boolean result = CentralRepoDbUtil.insertDefaultCorrelationTypes(conn) + && CentralRepoDbUtil.insertDefaultOrganization(conn) + && insertDefaultPersonaTablesContent(conn); + + CentralRepoDbUtil.closeConnection(conn); return result; } @@ -413,7 +424,7 @@ public class RdbmsCentralRepoFactory { /** * Get the template for creating an index on the value column of an instance * table. %s will exist in the template where the name of the new table will - * be addedd. + * be added. * * @return a String which is a template for adding an index to the value * column of a _instances table @@ -426,7 +437,7 @@ public class RdbmsCentralRepoFactory { /** * Get the template for creating an index on the known_status column of an * instance table. %s will exist in the template where the name of the new - * table will be addedd. + * table will be added. * * @return a String which is a template for adding an index to the * known_status column of a _instances table @@ -439,7 +450,7 @@ public class RdbmsCentralRepoFactory { /** * Get the template for creating an index on the file_obj_id column of an * instance table. %s will exist in the template where the name of the new - * table will be addedd. + * table will be added. * * @return a String which is a template for adding an index to the * file_obj_id column of a _instances table @@ -525,7 +536,16 @@ public class RdbmsCentralRepoFactory { } } - + private static String getOnConflictDoNothingClause(CentralRepoPlatforms selectedPlatform) { + switch (selectedPlatform) { + case POSTGRESQL: + return "ON CONFLICT DO NOTHING"; + case SQLITE: + return ""; + default: + return ""; + } + } /** * Returns an ephemeral connection to the CR database. * @@ -541,4 +561,310 @@ public class RdbmsCentralRepoFactory { return null; } } + + /** + * Creates the tables for Persona. + * + * @return True if success, False otherwise. + */ + private boolean createPersonaTables(Statement stmt) throws SQLException { + + stmt.execute(getCreateAccountTypesTableStatement(selectedPlatform)); + stmt.execute(getCreateConfidenceTableStatement(selectedPlatform)); + stmt.execute(getCreateExaminersTableStatement(selectedPlatform)); + stmt.execute(getCreatePersonaStatusTableStatement(selectedPlatform)); + stmt.execute(getCreateAliasesTableStatement(selectedPlatform)); + + stmt.execute(getCreateAccountsTableStatement(selectedPlatform)); + stmt.execute(getCreatePersonasTableStatement(selectedPlatform)); + stmt.execute(getCreatePersonaAliasTableStatement(selectedPlatform)); + stmt.execute(getCreatePersonaMetadataTableStatement(selectedPlatform)); + stmt.execute(getCreatePersonaAccountsTableStatement(selectedPlatform)); + + return true; + } + + + /** + * Get the SQL string for creating a new account_types table in a central + * repository. + * + * @return SQL string for creating account_types table + */ + static String getCreateAccountTypesTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS account_types (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "type_name TEXT NOT NULL," + + "display_name TEXT NOT NULL," + + "correlation_type_id " + getBigIntType(selectedPlatform) + " ," + + "CONSTRAINT type_name_unique UNIQUE (type_name)," + + "FOREIGN KEY (correlation_type_id) REFERENCES correlation_types(id)" + + ")"; + } + + /** + * Get the SQL String for creating a new confidence table in a central + * repository. + * + * @return SQL string for creating confidence table + */ + static String getCreateConfidenceTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS confidence (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "confidence_id integer NOT NULL," + + "description TEXT," + + "CONSTRAINT level_unique UNIQUE (confidence_id)" + + ")"; + } + + /** + * Get the SQL String for creating a new examiners table in a central + * repository. + * + * @return SQL string for creating examiners table + */ + static String getCreateExaminersTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS examiners (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "login_name TEXT NOT NULL," + + "display_name TEXT," + + "CONSTRAINT login_name_unique UNIQUE(login_name)" + + ")"; + } + + /** + * Get the SQL String for creating a new persona_status table in a central + * repository. + * + * @return SQL string for creating persona_status table + */ + static String getCreatePersonaStatusTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS persona_status (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "status_id integer NOT NULL," + + "status TEXT NOT NULL," + + "CONSTRAINT status_unique UNIQUE(status_id)" + + ")"; + } + + /** + * Get the SQL String for creating a new aliases table in a central + * repository. + * + * @return SQL string for creating aliases table + */ + static String getCreateAliasesTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS aliases (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "alias TEXT NOT NULL," + + "CONSTRAINT alias_unique UNIQUE(alias)" + + ")"; + } + + /** + * Get the SQL String for creating a new accounts table in a central + * repository. + * + * @return SQL string for creating accounts table + */ + static String getCreateAccountsTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS accounts (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "account_type_id integer NOT NULL," + + "account_unique_identifier TEXT NOT NULL," + + "CONSTRAINT account_unique UNIQUE(account_type_id, account_unique_identifier)," + + "FOREIGN KEY (account_type_id) REFERENCES account_types(id)" + + ")"; + } + + /** + * Get the SQL String for creating a new personas table in a central + * repository. + * + * @return SQL string for creating personas table + */ + static String getCreatePersonasTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS personas (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "uuid TEXT NOT NULL," + + "comment TEXT NOT NULL," + + "name TEXT NOT NULL," + + "created_date " + getBigIntType(selectedPlatform) + " ," + + "modified_date " + getBigIntType(selectedPlatform) + " ," + + "status_id integer NOT NULL," + + "CONSTRAINT uuid_unique UNIQUE(uuid)," + + "FOREIGN KEY (status_id) REFERENCES persona_status(status_id)" + + ")"; + } + + /** + * Get the SQL String for creating a new persona_alias table in a central + * repository. + * + * @return SQL string for creating persona_alias table + */ + static String getCreatePersonaAliasTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS persona_alias (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "persona_id " + getBigIntType(selectedPlatform) + " ," + + "alias_id " + getBigIntType(selectedPlatform) + " ," + + "justification TEXT NOT NULL," + + "confidence_id integer NOT NULL," + + "date_added " + getBigIntType(selectedPlatform) + " ," + + "examiner_id integer NOT NULL," + + "FOREIGN KEY (persona_id) REFERENCES personas(id)," + + "FOREIGN KEY (alias_id) REFERENCES aliases(id)," + + "FOREIGN KEY (confidence_id) REFERENCES confidence(confidence_id)," + + "FOREIGN KEY (examiner_id) REFERENCES examiners(id)" + + ")"; + } + + /** + * Get the SQL String for creating a new persona_metadata table in a central + * repository. + * + * @return SQL string for creating persona_metadata table + */ + static String getCreatePersonaMetadataTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS persona_metadata (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "persona_id " + getBigIntType(selectedPlatform) + " ," + + "name TEXT NOT NULL," + + "value TEXT NOT NULL," + + "justification TEXT NOT NULL," + + "confidence_id integer NOT NULL," + + "date_added " + getBigIntType(selectedPlatform) + " ," + + "examiner_id integer NOT NULL," + + "CONSTRAINT unique_metadata UNIQUE(persona_id, name)," + + "FOREIGN KEY (persona_id) REFERENCES personas(id)," + + "FOREIGN KEY (confidence_id) REFERENCES confidence(confidence_id)," + + "FOREIGN KEY (examiner_id) REFERENCES examiners(id)" + + ")"; + } + + /** + * Get the SQL String for creating a new persona_accounts table in a central + * repository. + * + * @return SQL string for creating persona_accounts table + */ + static String getCreatePersonaAccountsTableStatement(CentralRepoPlatforms selectedPlatform) { + + return "CREATE TABLE IF NOT EXISTS persona_accounts (" + + getNumericPrimaryKeyClause("id", selectedPlatform) + + "persona_id " + getBigIntType(selectedPlatform) + " ," + + "account_id " + getBigIntType(selectedPlatform) + " ," + + "justification TEXT NOT NULL," + + "confidence_id integer NOT NULL," + + "date_added " + getBigIntType(selectedPlatform) + " ," + + "examiner_id integer NOT NULL," + + "FOREIGN KEY (persona_id) REFERENCES personas(id)," + + "FOREIGN KEY (account_id) REFERENCES accounts(id)," + + "FOREIGN KEY (confidence_id) REFERENCES confidence(confidence_id)," + + "FOREIGN KEY (examiner_id) REFERENCES examiners(id)" + + ")"; + } + + + /** + * Inserts the default content in persona related tables. + * + * @param conn Database connection to use. + * + * @return True if success, false otherwise. + */ + private boolean insertDefaultPersonaTablesContent(Connection conn) { + + Statement stmt = null; + try { + stmt = conn.createStatement(); + + // populate the confidence table + for (Confidence confidence : Persona.Confidence.values()) { + String sqlString = "INSERT INTO confidence (confidence_id, description) VALUES ( " + confidence.getLevel() + ", '" + confidence.toString() + "')" //NON-NLS + + getOnConflictDoNothingClause(selectedPlatform); + stmt.execute(sqlString); + } + + // populate the persona_status table + for (PersonaStatus status : Persona.PersonaStatus.values()) { + String sqlString = "INSERT INTO persona_status (status_id, status) VALUES ( " + status.getStatus() + ", '" + status.toString() + "')" //NON-NLS + + getOnConflictDoNothingClause(selectedPlatform); + stmt.execute(sqlString); + } + + // Populate the account_types table + for (Account.Type type : Account.Type.PREDEFINED_ACCOUNT_TYPES) { + int correlationTypeId = getCorrelationTypeIdForAccountType(conn, type); + if (correlationTypeId > 0) { + String sqlString = String.format("INSERT INTO account_types (type_name, display_name, correlation_type_id) VALUES ('%s', '%s', %d)" + getOnConflictDoNothingClause(selectedPlatform), + type.getTypeName(), type.getDisplayName(), correlationTypeId); + stmt.execute(sqlString); + } + } + + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, String.format("Failed to populate default data in Persona tables."), ex); + return false; + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException ex2) { + LOGGER.log(Level.SEVERE, "Error closing statement.", ex2); + } + } + } + + return true; + } + + /** + * Returns the correlation type id for the given account type, + * from the correlation_types table. + * + * @param conn Connection to use for database query. + * @param accountType Account type to look for. + * ' + * @return correlation type id. + */ + private int getCorrelationTypeIdForAccountType(Connection conn, Account.Type accountType) { + + int typeId = -1; + if (accountType == Account.Type.EMAIL) { + typeId = CorrelationAttributeInstance.EMAIL_TYPE_ID; + } else if (accountType == Account.Type.PHONE) { + typeId = CorrelationAttributeInstance.PHONE_TYPE_ID; + } else { + ResultSet resultSet = null; + + PreparedStatement preparedStatementQuery = null; + String querySql = "SELECT * FROM correlation_types WHERE display_name=?"; + try { + preparedStatementQuery = conn.prepareStatement(querySql); + preparedStatementQuery.setString(1, accountType.getDisplayName()); + + resultSet = preparedStatementQuery.executeQuery(); + if (resultSet.next()) { + typeId = resultSet.getInt("id"); + } + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, String.format("Failed to get correlation typeId for account type %s.", accountType.getTypeName()), ex); + } finally { + CentralRepoDbUtil.closeStatement(preparedStatementQuery); + CentralRepoDbUtil.closeResultSet(resultSet); + } + } + + return typeId; + } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java index 5c5ba6072b..519a7b2453 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java @@ -460,7 +460,6 @@ public class EamDbSettingsDialog extends JDialog { if (!result) { // Remove the incomplete database if (dbCreated) { - // RAMAN TBD: migrate deleteDatabase() to RdbmsCentralRepoFactory dbSettingsPostgres.deleteDatabase(); } diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java index 0f87533504..e096638434 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018-2019 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -123,8 +123,7 @@ final public class CommonAttributeCaseSearchResults { if (currentCaseDataSourceMap == null) { //there are no results return filteredCaseNameToDataSourcesTree; } - CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance - .getDefaultCorrelationTypes() + CorrelationAttributeInstance.Type attributeType = CentralRepository.getInstance().getCorrelationTypes() .stream() .filter(filterType -> filterType.getId() == resultTypeId) .findFirst().get(); diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java index 18e7195f34..85923e53b6 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java @@ -128,14 +128,13 @@ final public class CommonAttributeCountSearchResults { return; } - CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance - .getDefaultCorrelationTypes() + CentralRepository eamDb = CentralRepository.getInstance(); + CorrelationAttributeInstance.Type attributeType = eamDb.getCorrelationTypes() .stream() .filter(filterType -> filterType.getId() == this.resultTypeId) .findFirst().get(); - CentralRepository eamDb = CentralRepository.getInstance(); - + Map> itemsToRemove = new HashMap<>(); //Call countUniqueDataSources once to reduce the number of DB queries needed to get //the frequencyPercentage diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributePanel.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributePanel.java index f56f9dd9a8..43d834e39d 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributePanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributePanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -255,7 +255,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer filterByDocuments = interCasePanel.documentsCheckboxIsSelected(); } if (corType == null) { - corType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0); + corType = CentralRepository.getInstance().getCorrelationTypes().get(0); } if (caseId == InterCasePanel.NO_CASE_SELECTED) { builder = new AllInterCaseCommonAttributeSearcher(filterByMedia, filterByDocuments, corType, percentageThreshold); @@ -366,7 +366,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer filterByDocuments = interCasePanel.documentsCheckboxIsSelected(); } if (corType == null) { - corType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0); + corType = CentralRepository.getInstance().getCorrelationTypes().get(0); } if (caseId == InterCasePanel.NO_CASE_SELECTED) { builder = new AllInterCaseCommonAttributeSearcher(filterByMedia, filterByDocuments, corType, percentageThreshold); diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCasePanel.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCasePanel.java index 88d60db3be..a2c1c01529 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCasePanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCasePanel.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018-2019 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,6 +31,7 @@ import java.util.logging.Level; import javax.swing.ComboBoxModel; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException; +import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; import org.sleuthkit.autopsy.coreutils.Logger; /** @@ -117,7 +118,7 @@ public final class InterCasePanel extends javax.swing.JPanel { void setupCorrelationTypeFilter() { this.correlationTypeFilters = new HashMap<>(); try { - List types = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + List types = CentralRepository.getInstance().getCorrelationTypes(); for (CorrelationAttributeInstance.Type type : types) { correlationTypeFilters.put(type.getDisplayName(), type); this.correlationTypeComboBox.addItem(type.getDisplayName()); diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/CorrelationCaseChildNodeFactory.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/CorrelationCaseChildNodeFactory.java index 5105f1628d..49617c9dfb 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/CorrelationCaseChildNodeFactory.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/CorrelationCaseChildNodeFactory.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2019 Basis Technology Corp. + * Copyright 2019-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -111,7 +111,7 @@ final class CorrelationCaseChildNodeFactory extends ChildFactory(); - List correcationTypeList = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + List correcationTypeList = CentralRepository.getInstance().getCorrelationTypes(); correcationTypeList.forEach((type) -> { correlationTypeMap.put(type.getId(), type); }); diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java index 69dcdc71b9..65b47bcb3b 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018-2019 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,6 +62,7 @@ import org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFa import org.sleuthkit.autopsy.modules.vmextractor.VMExtractorIngestModuleFactory; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; +import org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepoFactory; /** * Utilities for testing intercase correlation feature. @@ -220,7 +221,7 @@ class InterCaseTestUtils { this.kitchenShink = new IngestJobSettings(InterCaseTestUtils.class.getCanonicalName(), IngestType.ALL_MODULES, kitchenSink); try { - Collection types = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + Collection types = CentralRepository.getInstance().getCorrelationTypes(); //TODO use ids instead of strings FILE_TYPE = types.stream().filter(type -> type.getDisplayName().equals("Files")).findAny().get(); @@ -248,7 +249,7 @@ class InterCaseTestUtils { CentralRepository.getInstance().shutdownConnections(); } FileUtils.deleteDirectory(CENTRAL_REPO_DIRECTORY_PATH.toFile()); - } catch (IOException | CentralRepoExceptionex) { + } catch (IOException | CentralRepoException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); } @@ -297,8 +298,10 @@ class InterCaseTestUtils { crSettings.createDbDirectory(); } - crSettings.initializeDatabaseSchema(); - crSettings.insertDefaultDatabaseContent(); + RdbmsCentralRepoFactory centralRepoSchemaFactory = new RdbmsCentralRepoFactory(CentralRepoPlatforms.SQLITE, crSettings); + centralRepoSchemaFactory.initializeDatabaseSchema(); + centralRepoSchemaFactory.insertDefaultDatabaseContent(); + crSettings.saveSettings(); CentralRepoPlatforms.setSelectedPlatform(CentralRepoPlatforms.SQLITE.name()); CentralRepoPlatforms.saveSelectedPlatform(); From 29c3cb6c25720abe750f3a323587a0316953116b Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Mon, 24 Feb 2020 14:16:37 -0500 Subject: [PATCH 50/59] Addressed Codacy comment. --- .../autopsy/centralrepository/datamodel/CentralRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java index e357d529e5..fe54161762 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java @@ -811,5 +811,5 @@ public interface CentralRepository { * @return list of Correlation types * @throws CentralRepoException */ - public List getCorrelationTypes() throws CentralRepoException; + List getCorrelationTypes() throws CentralRepoException; } From c8bf39e715490170196add63610ae8ad7508ff35 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 24 Feb 2020 16:00:31 -0500 Subject: [PATCH 51/59] Skeleton for creating correlation attrs for account artifacts --- .../DataContentViewerOtherCases.java | 4 +- .../datamodel/CorrelationAttributeUtil.java | 115 ++++++++++-------- .../eventlisteners/CaseEventListener.java | 4 +- .../eventlisteners/IngestEventsListener.java | 4 +- .../AnnotationsContentViewer.java | 4 +- .../autopsy/datamodel/GetSCOTask.java | 2 +- 6 files changed, 74 insertions(+), 59 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index a28a013220..027a4ad7df 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2017-2019 Basis Technology Corp. + * Copyright 2017-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -464,7 +464,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi // correlate on blackboard artifact attributes if they exist and supported BlackboardArtifact bbArtifact = getBlackboardArtifactFromNode(node); if (bbArtifact != null && CentralRepository.isEnabled()) { - ret.addAll(CorrelationAttributeUtil.makeInstancesFromBlackboardArtifact(bbArtifact, false)); + ret.addAll(CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact)); } // we can correlate based on the MD5 if it is enabled diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index 07eb454ac5..9ac0dddf9e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2020 Basis Technology Corp. + * Copyright 2017-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,55 +48,61 @@ public class CorrelationAttributeUtil { } /** - * Static factory method to examine a BlackboardArtifact to determine if it - * has contents that can be used for Correlation. If so, return a - * EamArtifact with a single EamArtifactInstance within. If not, return - * null. + * Examines an artifact and makes zero to many correlation attribute + * instances from its attributes. * - * @param artifact BlackboardArtifact to examine - * @param checkEnabled If true, only create a CorrelationAttribute if it is - * enabled + * IMPORTANT: The correlation attribute instances are NOT added to the + * central repository by this method. * - * @return List of EamArtifacts + * @param artifact An artifact. + * + * @return A list, possibly empty, of correlation attribute instances for + * the artifact. */ - public static List makeInstancesFromBlackboardArtifact(BlackboardArtifact artifact, - boolean checkEnabled) { - List eamArtifacts = new ArrayList<>(); + public static List makeAttrsForArtifact(BlackboardArtifact artifact) { + List correlationAttrs = new ArrayList<>(); try { - BlackboardArtifact artifactForInstance = null; + /* + * If the artifact is an interesting artifact hit, examine the + * interesting artifact, not the hit "meta-artifact." + */ + BlackboardArtifact artToExamine = null; if (BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT.getTypeID() == artifact.getArtifactTypeID()) { - // Get the associated artifactForInstance - BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT)); - if (attribute != null) { - artifactForInstance = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboardArtifact(attribute.getValueLong()); + BlackboardAttribute assocArtifactAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT)); + if (assocArtifactAttr != null) { + artToExamine = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboardArtifact(assocArtifactAttr.getValueLong()); } } else { - artifactForInstance = artifact; + artToExamine = artifact; } - if (artifactForInstance != null) { - int artifactTypeID = artifactForInstance.getArtifactTypeID(); + + /* + * + */ + if (artToExamine != null) { + int artifactTypeID = artToExamine.getArtifactTypeID(); if (artifactTypeID == ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) { - BlackboardAttribute setNameAttr = artifactForInstance.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); - if (setNameAttr != null - && CorrelationAttributeUtil.getEmailAddressAttrString().equals(setNameAttr.getValueString())) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID); + BlackboardAttribute setNameAttr = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); + if (setNameAttr != null && CorrelationAttributeUtil.getEmailAddressAttrString().equals(setNameAttr.getValueString())) { + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID); } + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID()) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN, CorrelationAttributeInstance.DOMAIN_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN, CorrelationAttributeInstance.DOMAIN_TYPE_ID); + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_CONTACT.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_CALLLOG.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) { - String value = null; - if (null != artifactForInstance.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER))) { - value = artifactForInstance.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER)).getValueString(); - } else if (null != artifactForInstance.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM))) { - value = artifactForInstance.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM)).getValueString(); - } else if (null != artifactForInstance.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO))) { - value = artifactForInstance.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO)).getValueString(); + if (null != artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER))) { + value = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER)).getValueString(); + } else if (null != artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM))) { + value = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM)).getValueString(); + } else if (null != artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO))) { + value = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO)).getValueString(); } // Remove all non-numeric symbols to semi-normalize phone numbers, preserving leading "+" character if (value != null) { @@ -108,44 +114,53 @@ public class CorrelationAttributeUtil { // Only add the correlation attribute if the resulting phone number large enough to be of use // (these 3-5 digit numbers can be valid, but are not useful for correlation) if (value.length() > 5) { - CorrelationAttributeInstance inst = makeCorrelationAttributeInstanceUsingTypeValue(artifactForInstance, CentralRepository.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.PHONE_TYPE_ID), value); + CorrelationAttributeInstance inst = makeCorrelationAttributeInstanceUsingTypeValue(artToExamine, CentralRepository.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.PHONE_TYPE_ID), value); if (inst != null) { - eamArtifacts.add(inst); + correlationAttrs.add(inst); } } } + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID()) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID, CorrelationAttributeInstance.USBID_TYPE_ID); - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID, CorrelationAttributeInstance.USBID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID()) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID, CorrelationAttributeInstance.SSID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID, CorrelationAttributeInstance.SSID_TYPE_ID); + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID()) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID()) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI, CorrelationAttributeInstance.IMEI_TYPE_ID); - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI, CorrelationAttributeInstance.IMEI_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID()) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID()) { - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, CorrelationAttributeInstance.PHONE_TYPE_ID); - addCorrelationAttributeToList(eamArtifacts, artifactForInstance, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL, CorrelationAttributeInstance.EMAIL_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, CorrelationAttributeInstance.PHONE_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL, CorrelationAttributeInstance.EMAIL_TYPE_ID); + + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID()) { + // RJCTODO: Make a correlation attribute by switching on account type } } } catch (CentralRepoException ex) { logger.log(Level.SEVERE, "Error getting defined correlation types.", ex); // NON-NLS - return eamArtifacts; + return correlationAttrs; } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS - return null; + return correlationAttrs; } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); // NON-NLS - return null; + return correlationAttrs; } - return eamArtifacts; + return correlationAttrs; } /** @@ -166,7 +181,7 @@ public class CorrelationAttributeUtil { * @throws CentralRepoException * @throws TskCoreException */ - private static void addCorrelationAttributeToList(List eamArtifacts, BlackboardArtifact artifact, ATTRIBUTE_TYPE bbAttributeType, int typeId) throws CentralRepoException, TskCoreException { + private static void makeCorrAttrFromArtifactAttr(List eamArtifacts, BlackboardArtifact artifact, ATTRIBUTE_TYPE bbAttributeType, int typeId) throws CentralRepoException, TskCoreException { BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(bbAttributeType)); if (attribute != null) { String value = attribute.getValueString(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index c5f6ebdbe2..2775d5fe4f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -297,7 +297,7 @@ final class CaseEventListener implements PropertyChangeListener { return; } - List convertedArtifacts = CorrelationAttributeUtil.makeInstancesFromBlackboardArtifact(bbArtifact, true); + List convertedArtifacts = CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact); for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) { eamArtifact.setComment(comment); try { @@ -370,7 +370,7 @@ final class CaseEventListener implements PropertyChangeListener { if (!hasTagWithConflictingKnownStatus) { //Get the correlation atttributes that correspond to the current BlackboardArtifactTag if their status should be changed //with the initial set of correlation attributes this should be a single correlation attribute - List convertedArtifacts = CorrelationAttributeUtil.makeInstancesFromBlackboardArtifact(bbTag.getArtifact(), true); + List convertedArtifacts = CorrelationAttributeUtil.makeAttrsForArtifact(bbTag.getArtifact()); for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) { CentralRepository.getInstance().setAttributeInstanceKnownStatus(eamArtifact, tagName.getKnownStatus()); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 803f20b8c4..7ee644cbec 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2019 Basis Technology Corp. + * Copyright 2017-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -456,7 +456,7 @@ public class IngestEventsListener { for (BlackboardArtifact bbArtifact : bbArtifacts) { // eamArtifact will be null OR a EamArtifact containing one EamArtifactInstance. - List convertedArtifacts = CorrelationAttributeUtil.makeInstancesFromBlackboardArtifact(bbArtifact, true); + List convertedArtifacts = CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact); for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) { try { // Only do something with this artifact if it's unique within the job diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java index 9116ff9b45..9a320f97b7 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -198,7 +198,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data startSection(html, "Central Repository Comments"); List instancesList = new ArrayList<>(); if (artifact != null) { - instancesList.addAll(CorrelationAttributeUtil.makeInstancesFromBlackboardArtifact(artifact, false)); + instancesList.addAll(CorrelationAttributeUtil.makeAttrsForArtifact(artifact)); } try { List artifactTypes = CentralRepository.getInstance().getDefinedCorrelationTypes(); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/GetSCOTask.java b/Core/src/org/sleuthkit/autopsy/datamodel/GetSCOTask.java index 5f6f6f1c72..93d5694762 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/GetSCOTask.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/GetSCOTask.java @@ -97,7 +97,7 @@ class GetSCOTask implements Runnable { logger.log(Level.WARNING, "Unable to get correlation type or value to determine value for O column for artifact", ex); } } else { - List listOfPossibleAttributes = CorrelationAttributeUtil.makeInstancesFromBlackboardArtifact(bbArtifact, false); + List listOfPossibleAttributes = CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact); if (listOfPossibleAttributes.size() > 1) { //Don't display anything if there is more than 1 correlation property for an artifact but let the user know description = Bundle.GetSCOTask_occurrences_multipleProperties(); From b1050b96700190e731dc3eaa23b5bf2503b19a85 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 24 Feb 2020 16:14:16 -0500 Subject: [PATCH 52/59] Skeleton for creating correlation attrs for account artifacts --- .../datamodel/CorrelationAttributeUtil.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index 9ac0dddf9e..b241495b64 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -77,16 +77,16 @@ public class CorrelationAttributeUtil { } /* - * + * */ if (artToExamine != null) { - int artifactTypeID = artToExamine.getArtifactTypeID(); + int artifactTypeID = artToExamine.getArtifactTypeID(); if (artifactTypeID == ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) { BlackboardAttribute setNameAttr = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); if (setNameAttr != null && CorrelationAttributeUtil.getEmailAddressAttrString().equals(setNameAttr.getValueString())) { makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID); } - + } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID() @@ -147,7 +147,7 @@ public class CorrelationAttributeUtil { makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL, CorrelationAttributeInstance.EMAIL_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID()) { - // RJCTODO: Make a correlation attribute by switching on account type + makeAttributeFromAccountArtifact(correlationAttrs, artToExamine); } } } catch (CentralRepoException ex) { @@ -163,6 +163,21 @@ public class CorrelationAttributeUtil { return correlationAttrs; } + /** + * Makes a correlation attribute instance for an account artifact. + * + * IMPORTANT: The correlation attribute instance is NOT added to the central + * repository by this method. + * + * @param acctArtifact An account artifact. + */ + private static CorrelationAttributeInstance makeAttributeFromAccountArtifact(List correlationAttrs, BlackboardArtifact acctArtifact) { + // TODO: Convert TSK_ACCOUNT_TYPE attribute to correlation attribute type + // TODO: Extract TSK_ID as value + // return makeCorrelationAttributeInstanceUsingTypeValue(acctArtifact, null, ""); + return null; + } + /** * Add a CorrelationAttributeInstance of the specified type to the provided * list if the artifactForInstance has an Attribute of the given type with a From 75e55dc7d4e96dd4b329413ba046fb12f05c5e73 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 24 Feb 2020 16:36:05 -0500 Subject: [PATCH 53/59] Skeleton for creating correlation attrs for account artifacts --- .../datamodel/CorrelationAttributeUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index b241495b64..ab1a381720 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -151,13 +151,13 @@ public class CorrelationAttributeUtil { } } } catch (CentralRepoException ex) { - logger.log(Level.SEVERE, "Error getting defined correlation types.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting defined correlation types", ex); // NON-NLS return correlationAttrs; } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting querying case database for artifact attribute", ex); // NON-NLS return correlationAttrs; } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting current case", ex); // NON-NLS return correlationAttrs; } return correlationAttrs; From b78aa977c2672d9c492d144a22d6c4923ba3fb58 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Tue, 25 Feb 2020 14:46:33 -0500 Subject: [PATCH 54/59] 6014: Create accounts & persona tables - Added an account_id column to X_instances tables. --- .../datamodel/RdbmsCentralRepoFactory.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java index 3ef09b263c..f61fb42062 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java @@ -136,6 +136,10 @@ public class RdbmsCentralRepoFactory { stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + // Create account_types and accounts tab;es which are referred by X_instances tables + stmt.execute(getCreateAccountTypesTableStatement(selectedPlatform)); + stmt.execute(getCreateAccountsTableStatement(selectedPlatform)); + // Create a separate instance and reference table for each artifact type List defaultCorrelationTypes = CorrelationAttributeInstance.getDefaultCorrelationTypes(); @@ -364,12 +368,14 @@ public class RdbmsCentralRepoFactory { + getNumericPrimaryKeyClause("id", selectedPlatform) + "case_id integer NOT NULL," + "data_source_id integer NOT NULL," + + "account_id " + getBigIntType(selectedPlatform) + " DEFAULT NULL," + "value text NOT NULL," + "file_path text NOT NULL," + "known_status integer NOT NULL," + "comment text," + "file_obj_id " + getBigIntType(selectedPlatform) + " ," + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path)" + getOnConflictIgnoreClause(selectedPlatform) + "," + + "foreign key (account_id) references accounts(id)," + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; } @@ -569,13 +575,11 @@ public class RdbmsCentralRepoFactory { */ private boolean createPersonaTables(Statement stmt) throws SQLException { - stmt.execute(getCreateAccountTypesTableStatement(selectedPlatform)); stmt.execute(getCreateConfidenceTableStatement(selectedPlatform)); stmt.execute(getCreateExaminersTableStatement(selectedPlatform)); stmt.execute(getCreatePersonaStatusTableStatement(selectedPlatform)); stmt.execute(getCreateAliasesTableStatement(selectedPlatform)); - stmt.execute(getCreateAccountsTableStatement(selectedPlatform)); stmt.execute(getCreatePersonasTableStatement(selectedPlatform)); stmt.execute(getCreatePersonaAliasTableStatement(selectedPlatform)); stmt.execute(getCreatePersonaMetadataTableStatement(selectedPlatform)); From 61cfc91c8708c5e2cab8fb21789a1fb633796daf Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 25 Feb 2020 19:50:43 -0500 Subject: [PATCH 55/59] Partial clean up of CorrelationAttributeUtil.java --- .../AddEditCentralRepoCommentAction.java | 4 +- .../DataContentViewerOtherCases.java | 2 +- .../datamodel/CorrelationAttributeUtil.java | 399 +++++++++++------- .../eventlisteners/CaseEventListener.java | 25 +- .../eventlisteners/IngestEventsListener.java | 2 +- .../AnnotationsContentViewer.java | 2 +- .../datamodel/AbstractAbstractFileNode.java | 2 +- .../datamodel/BlackboardArtifactNode.java | 4 +- .../autopsy/datamodel/GetSCOTask.java | 2 +- 9 files changed, 264 insertions(+), 178 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index dae8bbe312..5427b7b77a 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -61,10 +61,10 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { */ public AddEditCentralRepoCommentAction(AbstractFile file) { fileId = file.getId(); - correlationAttributeInstance = CorrelationAttributeUtil.getInstanceFromContent(file); + correlationAttributeInstance = CorrelationAttributeUtil.getCorrAttrForFile(file); if (correlationAttributeInstance == null) { addToDatabase = true; - correlationAttributeInstance = CorrelationAttributeUtil.makeInstanceFromContent(file); + correlationAttributeInstance = CorrelationAttributeUtil.makeCorrAttrFromFile(file); } if (file.getSize() == 0) { putValue(Action.NAME, Bundle.AddEditCentralRepoCommentAction_menuItemText_addEditCentralRepoCommentEmptyFile()); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 027a4ad7df..881e60236e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -464,7 +464,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi // correlate on blackboard artifact attributes if they exist and supported BlackboardArtifact bbArtifact = getBlackboardArtifactFromNode(node); if (bbArtifact != null && CentralRepository.isEnabled()) { - ret.addAll(CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact)); + ret.addAll(CorrelationAttributeUtil.makeCorrAttrsFromArtifact(bbArtifact)); } // we can correlate based on the MD5 if it is enabled diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index ab1a381720..a93354901b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -30,131 +30,110 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; -import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** - * Utility class for correlation attributes in the central repository + * Utility class for working with correlation attributes in the central + * repository. */ public class CorrelationAttributeUtil { private static final Logger logger = Logger.getLogger(CorrelationAttributeUtil.class.getName()); - @Messages({"EamArtifactUtil.emailaddresses.text=Email Addresses"}) - public static String getEmailAddressAttrString() { - return Bundle.EamArtifactUtil_emailaddresses_text(); + /** + * Gets a string that is expected to be the same string that is stored in + * the correlation_types table in the central repository as the display name + * for the email address correlation attribute type. This string is + * duplicated in the CorrelationAttributeInstance class. + * + * TODO (Jira-6088): We should not have multiple deifnitions of this string. + * + * @return The display name of the email address correlation attribute type. + */ + @Messages({"CorrelationAttributeUtil.emailaddresses.text=Email Addresses"}) + private static String getEmailAddressAttrDisplayName() { + return Bundle.CorrelationAttributeUtil_emailaddresses_text(); } /** - * Examines an artifact and makes zero to many correlation attribute - * instances from its attributes. + * Makes zero to many correlation attribute instances from the attributes of + * an artifact. * * IMPORTANT: The correlation attribute instances are NOT added to the * central repository by this method. * + * TODO (Jira-6088): The methods in this low-level, utility class should + * throw exceptions instead of logging them. The reason for this is that the + * clients of the utility class, not the utility class itself, should be in + * charge of error handling policy, per the Autopsy Coding Standard. Note + * that clients of several of these methods currently cannot determine + * whether receiving a null return value is an error or not, plus null + * checking is easy to forget, while catching exceptions is enforced. + * * @param artifact An artifact. * * @return A list, possibly empty, of correlation attribute instances for * the artifact. */ - public static List makeAttrsForArtifact(BlackboardArtifact artifact) { + public static List makeCorrAttrsFromArtifact(BlackboardArtifact artifact) { List correlationAttrs = new ArrayList<>(); try { - /* - * If the artifact is an interesting artifact hit, examine the - * interesting artifact, not the hit "meta-artifact." - */ - BlackboardArtifact artToExamine = null; - if (BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT.getTypeID() == artifact.getArtifactTypeID()) { - BlackboardAttribute assocArtifactAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT)); - if (assocArtifactAttr != null) { - artToExamine = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboardArtifact(assocArtifactAttr.getValueLong()); - } - } else { - artToExamine = artifact; - } - - /* - * - */ - if (artToExamine != null) { - int artifactTypeID = artToExamine.getArtifactTypeID(); + BlackboardArtifact sourceArtifact = getCorrAttrSourceArtifact(artifact); + if (sourceArtifact != null) { + int artifactTypeID = sourceArtifact.getArtifactTypeID(); if (artifactTypeID == ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) { - BlackboardAttribute setNameAttr = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); - if (setNameAttr != null && CorrelationAttributeUtil.getEmailAddressAttrString().equals(setNameAttr.getValueString())) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID); + BlackboardAttribute setNameAttr = sourceArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); + if (setNameAttr != null && CorrelationAttributeUtil.getEmailAddressAttrDisplayName().equals(setNameAttr.getValueString())) { + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID); } } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID()) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN, CorrelationAttributeInstance.DOMAIN_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN, CorrelationAttributeInstance.DOMAIN_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_CONTACT.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_CALLLOG.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) { - String value = null; - if (null != artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER))) { - value = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER)).getValueString(); - } else if (null != artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM))) { - value = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM)).getValueString(); - } else if (null != artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO))) { - value = artToExamine.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO)).getValueString(); - } - // Remove all non-numeric symbols to semi-normalize phone numbers, preserving leading "+" character - if (value != null) { - String newValue = value.replaceAll("\\D", ""); - if (value.startsWith("+")) { - newValue = "+" + newValue; - } - value = newValue; - // Only add the correlation attribute if the resulting phone number large enough to be of use - // (these 3-5 digit numbers can be valid, but are not useful for correlation) - if (value.length() > 5) { - CorrelationAttributeInstance inst = makeCorrelationAttributeInstanceUsingTypeValue(artToExamine, CentralRepository.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.PHONE_TYPE_ID), value); - if (inst != null) { - correlationAttrs.add(inst); - } - } - } + makeCorrAttrFromArtifactPhoneAttr(sourceArtifact); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID()) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID, CorrelationAttributeInstance.USBID_TYPE_ID); - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID, CorrelationAttributeInstance.USBID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID()) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID, CorrelationAttributeInstance.SSID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID, CorrelationAttributeInstance.SSID_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID()) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS, CorrelationAttributeInstance.MAC_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID()) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI, CorrelationAttributeInstance.IMEI_TYPE_ID); - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI, CorrelationAttributeInstance.IMEI_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID()) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI, CorrelationAttributeInstance.IMSI_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ICCID, CorrelationAttributeInstance.ICCID_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID()) { - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, CorrelationAttributeInstance.PHONE_TYPE_ID); - makeCorrAttrFromArtifactAttr(correlationAttrs, artToExamine, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL, CorrelationAttributeInstance.EMAIL_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, CorrelationAttributeInstance.PHONE_TYPE_ID); + makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL, CorrelationAttributeInstance.EMAIL_TYPE_ID); } else if (artifactTypeID == ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID()) { - makeAttributeFromAccountArtifact(correlationAttrs, artToExamine); + makeCorrAttrFromAcctArtifact(correlationAttrs, sourceArtifact); } } } catch (CentralRepoException ex) { - logger.log(Level.SEVERE, "Error getting defined correlation types", ex); // NON-NLS + logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", artifact), ex); // NON-NLS return correlationAttrs; } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error getting querying case database for artifact attribute", ex); // NON-NLS + logger.log(Level.SEVERE, String.format("Error getting querying case database (%s)", artifact), ex); // NON-NLS return correlationAttrs; } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Error getting current case", ex); // NON-NLS @@ -163,73 +142,168 @@ public class CorrelationAttributeUtil { return correlationAttrs; } + /** + * Gets the associated artifact of a "meta-artifact" such as an interesting + * artifact hit artifact. + * + * @param artifact An artifact. + * + * @return The associated artifact if the input artifact is a + * "meta-artifact", otherwise the input artifact. + * + * @throws NoCurrentCaseException If there is no open case. + * @throws TskCoreException If there is an error querying thew case + * database. + */ + private static BlackboardArtifact getCorrAttrSourceArtifact(BlackboardArtifact artifact) throws NoCurrentCaseException, TskCoreException { + BlackboardArtifact sourceArtifact = null; + if (BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT.getTypeID() == artifact.getArtifactTypeID()) { + BlackboardAttribute assocArtifactAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT)); + if (assocArtifactAttr != null) { + sourceArtifact = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboardArtifact(assocArtifactAttr.getValueLong()); + } + } else { + sourceArtifact = artifact; + } + return sourceArtifact; + } + + /** + * Make a correlation attribute instance from a phone number attribute of an + * artifact. + * + * @param artifact An artifact with a phone number attribute. + * + * @return The correlation instance artifact or null, if the phone number is + * not a valid correlation attribute. + * + * @throws TskCoreException If there is an error querying the case + * database. + * @throws CentralRepoException If there is an error querying the central + * repository. + */ + private static CorrelationAttributeInstance makeCorrAttrFromArtifactPhoneAttr(BlackboardArtifact artifact) throws TskCoreException, CentralRepoException { + CorrelationAttributeInstance corrAttr = null; + + /* + * Extract the phone number from the artifact attribute. + */ + String value = null; + if (null != artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER))) { + value = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER)).getValueString(); + } else if (null != artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM))) { + value = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM)).getValueString(); + } else if (null != artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO))) { + value = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO)).getValueString(); + } + + /* + * Normalize the phone number. + */ + if (value != null) { + String newValue = value.replaceAll("\\D", ""); + if (value.startsWith("+")) { + newValue = "+" + newValue; + } + value = newValue; + + /* + * Validate the phone number. Three to five digit phone numbers may + * be valid, but they are too short to use as correlation + * attributes. + */ + if (value.length() > 5) { + corrAttr = makeCorrAttr(artifact, CentralRepository.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.PHONE_TYPE_ID), value); + } + } + + return corrAttr; + } + /** * Makes a correlation attribute instance for an account artifact. * * IMPORTANT: The correlation attribute instance is NOT added to the central * repository by this method. * - * @param acctArtifact An account artifact. + * TODO (Jira-6088): The methods in this low-level, utility class should + * throw exceptions instead of logging them. The reason for this is that the + * clients of the utility class, not the utility class itself, should be in + * charge of error handling policy, per the Autopsy Coding Standard. Note + * that clients of several of these methods currently cannot determine + * whether receiving a null return value is an error or not, plus null + * checking is easy to forget, while catching exceptions is enforced. + * + * @param corrAttrInstances A list of correlation attribute instances. + * @param acctArtifact An account artifact. + * + * @return The correlation attribute instance. */ - private static CorrelationAttributeInstance makeAttributeFromAccountArtifact(List correlationAttrs, BlackboardArtifact acctArtifact) { - // TODO: Convert TSK_ACCOUNT_TYPE attribute to correlation attribute type - // TODO: Extract TSK_ID as value - // return makeCorrelationAttributeInstanceUsingTypeValue(acctArtifact, null, ""); - return null; + private static void makeCorrAttrFromAcctArtifact(List corrAttrInstances, BlackboardArtifact acctArtifact) { + // RAMAN TODO: Convert TSK_ACCOUNT_TYPE attribute to correlation attribute type + // RAMAN TODO: Extract TSK_ID as value +// CorrelationAttributeInstance corrAttr = makeCorrAttr(acctArtifact, corrType, corrAttrValue); +// if (corrAttr != null) { +// corrAttrInstances.add(corrAttr); +// } } /** - * Add a CorrelationAttributeInstance of the specified type to the provided - * list if the artifactForInstance has an Attribute of the given type with a - * non empty value. + * Makes a correlation attribute instance from a specified attribute of an + * artifact. The correlation attribute instance is added to an input list. * - * @param eamArtifacts the list of CorrelationAttributeInstance objects - * which should be added to - * @param artifact the blackboard artifactForInstance which we are - * creating a CorrelationAttributeInstance for - * @param bbAttributeType the type of BlackboardAttribute we expect to exist - * for a CorrelationAttributeInstance of this type - * generated from this Blackboard Artifact - * @param typeId the integer type id of the - * CorrelationAttributeInstance type + * @param corrAttrInstances A list of correlation attribute instances. + * @param artifact An artifact. + * @param artAttrType The type of the atrribute of the artifact that + * is to be made into a correlatin attribute + * instance. + * @param typeId The type ID for the desired correlation + * attribute instance. * - * @throws CentralRepoException - * @throws TskCoreException + * @throws CentralRepoException If there is an error querying the central + * repository. + * @throws TskCoreException If there is an error querying the case + * database. */ - private static void makeCorrAttrFromArtifactAttr(List eamArtifacts, BlackboardArtifact artifact, ATTRIBUTE_TYPE bbAttributeType, int typeId) throws CentralRepoException, TskCoreException { - BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(bbAttributeType)); + private static void makeCorrAttrFromArtifactAttr(List corrAttrInstances, BlackboardArtifact artifact, ATTRIBUTE_TYPE artAttrType, int typeId) throws CentralRepoException, TskCoreException { + BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(artAttrType)); if (attribute != null) { String value = attribute.getValueString(); if ((null != value) && (value.isEmpty() == false)) { - CorrelationAttributeInstance inst = makeCorrelationAttributeInstanceUsingTypeValue(artifact, CentralRepository.getInstance().getCorrelationTypeById(typeId), value); + CorrelationAttributeInstance inst = makeCorrAttr(artifact, CentralRepository.getInstance().getCorrelationTypeById(typeId), value); if (inst != null) { - eamArtifacts.add(inst); + corrAttrInstances.add(inst); } } } } /** - * Uses the determined type and vallue, then looks up instance details to - * create proper CorrelationAttributeInstance. + * Makes a correlation attribute instance of a given type from an artifact. * - * @param bbArtifact the blackboard artifactForInstance - * @param correlationType the given type - * @param value the artifactForInstance value + * @param artifact The artifact. + * @param correlationType the correlation attribute type. + * @param value The correlation attribute value. * - * @return CorrelationAttributeInstance from details, or null if validation - * failed or another error occurred + * TODO (Jira-6088): The methods in this low-level, utility class should + * throw exceptions instead of logging them. The reason for this is that the + * clients of the utility class, not the utility class itself, should be in + * charge of error handling policy, per the Autopsy Coding Standard. Note + * that clients of several of these methods currently cannot determine + * whether receiving a null return value is an error or not, plus null + * checking is easy to forget, while catching exceptions is enforced. + * + * @return The correlation attribute instance or null, if an error occurred. */ - private static CorrelationAttributeInstance makeCorrelationAttributeInstanceUsingTypeValue(BlackboardArtifact bbArtifact, CorrelationAttributeInstance.Type correlationType, String value) { + private static CorrelationAttributeInstance makeCorrAttr(BlackboardArtifact artifact, CorrelationAttributeInstance.Type correlationType, String value) { try { Case currentCase = Case.getCurrentCaseThrows(); - AbstractFile bbSourceFile = currentCase.getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID()); + AbstractFile bbSourceFile = currentCase.getSleuthkitCase().getAbstractFileById(artifact.getObjectID()); if (null == bbSourceFile) { logger.log(Level.SEVERE, "Error creating artifact instance. Abstract File was null."); // NON-NLS return null; } - // make an instance for the BB source file CorrelationCase correlationCase = CentralRepository.getInstance().getCase(Case.getCurrentCaseThrows()); return new CorrelationAttributeInstance( correlationType, @@ -242,31 +316,34 @@ public class CorrelationAttributeUtil { bbSourceFile.getId()); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error getting AbstractFile for artifact: " + bbArtifact.toString(), ex); // NON-NLS + logger.log(Level.SEVERE, String.format("Error getting querying case database (%s)", artifact), ex); // NON-NLS return null; } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) { - logger.log(Level.WARNING, "Error creating artifact instance for artifact: " + bbArtifact.toString(), ex); // NON-NLS + logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", artifact), ex); // NON-NLS return null; } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Case is closed.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting current case", ex); // NON-NLS return null; } } /** - * Retrieve CorrelationAttribute from the given Content. + * Gets the correlation attribute instance for a file. * - * @param content The content object + * @param file The file. * - * @return The new CorrelationAttribute, or null if retrieval failed. + * TODO (Jira-6088): The methods in this low-level, utility class should + * throw exceptions instead of logging them. The reason for this is that the + * clients of the utility class, not the utility class itself, should be in + * charge of error handling policy, per the Autopsy Coding Standard. Note + * that clients of several of these methods currently cannot determine + * whether receiving a null return value is an error or not, plus null + * checking is easy to forget, while catching exceptions is enforced. + * + * @return The correlation attribute instance or null, if no such + * correlation attribute instance was found or an error occurred. */ - public static CorrelationAttributeInstance getInstanceFromContent(Content content) { - - if (!(content instanceof AbstractFile)) { - return null; - } - - final AbstractFile file = (AbstractFile) content; + public static CorrelationAttributeInstance getCorrAttrForFile(AbstractFile file) { if (!isSupportedAbstractFileType(file)) { return null; @@ -284,11 +361,14 @@ public class CorrelationAttributeUtil { return null; } correlationDataSource = CorrelationDataSource.fromTSKDataSource(correlationCase, file.getDataSource()); - } catch (TskCoreException | CentralRepoException ex) { - logger.log(Level.SEVERE, "Error retrieving correlation attribute.", ex); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Error getting querying case database (%s)", file), ex); // NON-NLS + return null; + } catch (CentralRepoException ex) { + logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", file), ex); // NON-NLS return null; } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Case is closed.", ex); + logger.log(Level.SEVERE, "Error getting current case", ex); // NON-NLS return null; } @@ -296,20 +376,22 @@ public class CorrelationAttributeUtil { try { correlationAttributeInstance = CentralRepository.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, file.getId()); } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) { - logger.log(Level.WARNING, String.format( - "Correlation attribute could not be retrieved for '%s' (id=%d): ", - content.getName(), content.getId()), ex); + logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", file), ex); // NON-NLS return null; } - //if there was no correlation attribute found for the item using object_id then check for attributes added with schema 1,1 which lack object_id + + /* + * If no correlation attribute instance was found when querying by file + * object ID, try searching by file path instead. This is necessary + * because file object IDs were not stored in the central repository in + * early versions of its schema. + */ if (correlationAttributeInstance == null && file.getMd5Hash() != null) { String filePath = (file.getParentPath() + file.getName()).toLowerCase(); try { correlationAttributeInstance = CentralRepository.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, file.getMd5Hash(), filePath); } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) { - logger.log(Level.WARNING, String.format( - "Correlation attribute could not be retrieved for '%s' (id=%d): ", - content.getName(), content.getId()), ex); + logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", file), ex); // NON-NLS return null; } } @@ -318,32 +400,31 @@ public class CorrelationAttributeUtil { } /** - * Create an EamArtifact from the given Content. Will return null if an - * artifactForInstance can not be created - this is not necessarily an error - * case, it just means an artifactForInstance can't be made. If creation - * fails due to an error (and not that the file is the wrong type or it has - * no hash), the error will be logged before returning. + * Makes a correlation attribute instance for a file. * - * Does not add the artifactForInstance to the database. + * IMPORTANT: The correlation attribute instance is NOT added to the central + * repository by this method. * - * @param content The content object + * TODO (Jira-6088): The methods in this low-level, utility class should + * throw exceptions instead of logging them. The reason for this is that the + * clients of the utility class, not the utility class itself, should be in + * charge of error handling policy, per the Autopsy Coding Standard. Note + * that clients of several of these methods currently cannot determine + * whether receiving a null return value is an error or not, plus null + * checking is easy to forget, while catching exceptions is enforced. * - * @return The new EamArtifact or null if creation failed + * @param file The file. + * + * @return The correlation attribute instance or null, if an error occurred. */ - public static CorrelationAttributeInstance makeInstanceFromContent(Content content) { + public static CorrelationAttributeInstance makeCorrAttrFromFile(AbstractFile file) { - if (!(content instanceof AbstractFile)) { + if (!isSupportedAbstractFileType(file)) { return null; } - final AbstractFile af = (AbstractFile) content; - - if (!isSupportedAbstractFileType(af)) { - return null; - } - - // We need a hash to make the artifactForInstance - String md5 = af.getMd5Hash(); + // We need a hash to make the correlation artifact instance. + String md5 = file.getMd5Hash(); if (md5 == null || md5.isEmpty() || HashUtility.isNoDataMd5(md5)) { return null; } @@ -354,31 +435,33 @@ public class CorrelationAttributeUtil { CorrelationCase correlationCase = CentralRepository.getInstance().getCase(Case.getCurrentCaseThrows()); return new CorrelationAttributeInstance( filesType, - af.getMd5Hash(), + file.getMd5Hash(), correlationCase, - CorrelationDataSource.fromTSKDataSource(correlationCase, af.getDataSource()), - af.getParentPath() + af.getName(), + CorrelationDataSource.fromTSKDataSource(correlationCase, file.getDataSource()), + file.getParentPath() + file.getName(), "", TskData.FileKnown.UNKNOWN, - af.getId()); + file.getId()); - } catch (TskCoreException | CentralRepoException | CorrelationAttributeNormalizationException ex) { - logger.log(Level.SEVERE, "Error making correlation attribute.", ex); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Error querying case database (%s)", file), ex); // NON-NLS + return null; + } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) { + logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", file), ex); // NON-NLS return null; } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Case is closed.", ex); + logger.log(Level.SEVERE, "Error getting current case", ex); // NON-NLS return null; } } /** - * Check whether the given abstract file should be processed for the central - * repository. + * Checks whether or not a file is of a type that can be added to the + * central repository as a correlation attribute instance. * - * @param file The file to test + * @param file A file. * - * @return true if the file should be added to the central repo, false - * otherwise + * @return True or false. */ public static boolean isSupportedAbstractFileType(AbstractFile file) { if (file == null) { @@ -405,9 +488,9 @@ public class CorrelationAttributeUtil { } /** - * Constructs a new EamArtifactUtil + * Prevent instantiation of this utility class. */ private CorrelationAttributeUtil() { - //empty constructor } + } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 2775d5fe4f..26c5271d76 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -197,7 +197,7 @@ final class CaseEventListener implements PropertyChangeListener { } } - final CorrelationAttributeInstance eamArtifact = CorrelationAttributeUtil.makeInstanceFromContent(af); + final CorrelationAttributeInstance eamArtifact = CorrelationAttributeUtil.makeCorrAttrFromFile(af); if (eamArtifact != null) { // send update to Central Repository db @@ -297,7 +297,7 @@ final class CaseEventListener implements PropertyChangeListener { return; } - List convertedArtifacts = CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact); + List convertedArtifacts = CorrelationAttributeUtil.makeCorrAttrsFromArtifact(bbArtifact); for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) { eamArtifact.setComment(comment); try { @@ -370,7 +370,7 @@ final class CaseEventListener implements PropertyChangeListener { if (!hasTagWithConflictingKnownStatus) { //Get the correlation atttributes that correspond to the current BlackboardArtifactTag if their status should be changed //with the initial set of correlation attributes this should be a single correlation attribute - List convertedArtifacts = CorrelationAttributeUtil.makeAttrsForArtifact(bbTag.getArtifact()); + List convertedArtifacts = CorrelationAttributeUtil.makeCorrAttrsFromArtifact(bbTag.getArtifact()); for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) { CentralRepository.getInstance().setAttributeInstanceKnownStatus(eamArtifact, tagName.getKnownStatus()); } @@ -406,9 +406,12 @@ final class CaseEventListener implements PropertyChangeListener { } //if the file will have no tags with a status which would prevent the current status from being changed if (!hasTagWithConflictingKnownStatus) { - final CorrelationAttributeInstance eamArtifact = CorrelationAttributeUtil.makeInstanceFromContent(contentTag.getContent()); - if (eamArtifact != null) { - CentralRepository.getInstance().setAttributeInstanceKnownStatus(eamArtifact, tagName.getKnownStatus()); + Content taggedContent = contentTag.getContent(); + if (taggedContent instanceof AbstractFile) { + final CorrelationAttributeInstance eamArtifact = CorrelationAttributeUtil.makeCorrAttrFromFile((AbstractFile)taggedContent); + if (eamArtifact != null) { + CentralRepository.getInstance().setAttributeInstanceKnownStatus(eamArtifact, tagName.getKnownStatus()); + } } } } @@ -455,7 +458,7 @@ final class CaseEventListener implements PropertyChangeListener { } } catch (CentralRepoException ex) { LOGGER.log(Level.SEVERE, "Error adding new data source to the central repository", ex); //NON-NLS - } + } } // DATA_SOURCE_ADDED } @@ -495,7 +498,7 @@ final class CaseEventListener implements PropertyChangeListener { } } // CURRENT_CASE } - + private final class DataSourceNameChangedTask implements Runnable { private final CentralRepository dbManager; @@ -508,12 +511,12 @@ final class CaseEventListener implements PropertyChangeListener { @Override public void run() { - + final DataSourceNameChangedEvent dataSourceNameChangedEvent = (DataSourceNameChangedEvent) event; Content dataSource = dataSourceNameChangedEvent.getDataSource(); String newName = (String) event.getNewValue(); - - if (! StringUtils.isEmpty(newName)) { + + if (!StringUtils.isEmpty(newName)) { if (!CentralRepository.isEnabled()) { return; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 7ee644cbec..e79f339c70 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -456,7 +456,7 @@ public class IngestEventsListener { for (BlackboardArtifact bbArtifact : bbArtifacts) { // eamArtifact will be null OR a EamArtifact containing one EamArtifactInstance. - List convertedArtifacts = CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact); + List convertedArtifacts = CorrelationAttributeUtil.makeCorrAttrsFromArtifact(bbArtifact); for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) { try { // Only do something with this artifact if it's unique within the job diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java index 9a320f97b7..3c967db3fd 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java @@ -198,7 +198,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data startSection(html, "Central Repository Comments"); List instancesList = new ArrayList<>(); if (artifact != null) { - instancesList.addAll(CorrelationAttributeUtil.makeAttrsForArtifact(artifact)); + instancesList.addAll(CorrelationAttributeUtil.makeCorrAttrsFromArtifact(artifact)); } try { List artifactTypes = CentralRepository.getInstance().getDefinedCorrelationTypes(); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 9bb2f8c358..441c5c6958 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -549,7 +549,7 @@ public abstract class AbstractAbstractFileNode extends A protected CorrelationAttributeInstance getCorrelationAttributeInstance() { CorrelationAttributeInstance attribute = null; if (CentralRepository.isEnabled() && !UserPreferences.getHideSCOColumns()) { - attribute = CorrelationAttributeUtil.getInstanceFromContent(content); + attribute = CorrelationAttributeUtil.getCorrAttrForFile(content); } return attribute; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 7c2b8c92ad..099d91449f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -605,8 +605,8 @@ public class BlackboardArtifactNode extends AbstractContentNode listOfPossibleAttributes = CorrelationAttributeUtil.makeAttrsForArtifact(bbArtifact); + List listOfPossibleAttributes = CorrelationAttributeUtil.makeCorrAttrsFromArtifact(bbArtifact); if (listOfPossibleAttributes.size() > 1) { //Don't display anything if there is more than 1 correlation property for an artifact but let the user know description = Bundle.GetSCOTask_occurrences_multipleProperties(); From 97a838653178f76167749a88aaac6dc4bdc49880 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 25 Feb 2020 20:04:30 -0500 Subject: [PATCH 56/59] Partial clean up of CorrelationAttributeUtil.java --- .../centralrepository/AddEditCentralRepoCommentAction.java | 2 +- .../centralrepository/datamodel/CorrelationAttributeUtil.java | 2 +- .../centralrepository/eventlisteners/CaseEventListener.java | 3 +-- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 2 +- .../sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index 5427b7b77a..5f47487f94 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index a93354901b..814169ef85 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -169,7 +169,7 @@ public class CorrelationAttributeUtil { } /** - * Make a correlation attribute instance from a phone number attribute of an + * Makes a correlation attribute instance from a phone number attribute of an * artifact. * * @param artifact An artifact with a phone number attribute. diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 26c5271d76..1df0e10dc6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2018 Basis Technology Corp. + * Copyright 2017-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,7 +52,6 @@ import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.TskDataException; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; /** diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 441c5c6958..a60964aa19 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2020 Basis Technology Corp. + * Copyright 2012-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 099d91449f..b58bb915d0 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2020 Basis Technology Corp. + * Copyright 2012-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); From 0c42694ec9830e7c1819083325d5cbcf66fd709d Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 26 Feb 2020 10:11:56 -0500 Subject: [PATCH 57/59] Added suppress deprecation warning to GeoFilterPanel --- Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java index e1c4e0773e..cca9382e51 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java @@ -52,6 +52,7 @@ class GeoFilterPanel extends javax.swing.JPanel { private final CheckBoxListPanel checkboxPanel; // Make sure to update if + @SuppressWarnings("deprecation") private static final BlackboardArtifact.ARTIFACT_TYPE[] GPS_ARTIFACT_TYPES = { BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK, BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_LAST_KNOWN_LOCATION, From 50789bceac9ef572f739e4c3a6d45a117d5af0a1 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 26 Feb 2020 10:30:29 -0500 Subject: [PATCH 58/59] Updated properties files --- .../sleuthkit/autopsy/geolocation/Bundle.properties-MERGED | 2 ++ .../sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED index 8e7439fb2f..e5bf351edb 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED @@ -30,6 +30,8 @@ GeoTopComponent_no_waypoints_returned_mgs=Applied filter failed to find waypoint GeoTopComponent_no_waypoints_returned_Title=No Waypoints Found GLTopComponent_initilzation_error=An error occurred during waypoint initilization. Geolocation data maybe incomplete. GLTopComponent_name=Geolocation +GLTopComponent_No_dataSource_message=There are no data sources with Geolocation artifacts found. +GLTopComponent_No_dataSource_Title=No Geolocation artifacts found HidingPane_default_title=Filters MapPanel_connection_failure_message=Failed to connect to new geolocation map tile source. MapPanel_connection_failure_message_title=Connection Failure diff --git a/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED index 23f2a02775..bf61ad9be0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED @@ -1,6 +1,6 @@ DATExtractor_process_message=Processing DJI DAT file: %s DATFileExtractor_Extractor_Name=DAT File Extractor -DroneIngestModule_Description=Description -DroneIngestModule_Name=Drone +DroneIngestModule_Description=Analyzes files generated by drones. +DroneIngestModule_Name=Drone Analyzer # {0} - AbstractFileName DroneIngestModule_process_start=Started {0} From 088cfaae6a79d7c9e381877d46cc91ba1a348bce Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Wed, 26 Feb 2020 13:02:23 -0500 Subject: [PATCH 59/59] Addressed review comments and Codacy comments. --- .../datamodel/RdbmsCentralRepo.java | 17 +- .../datamodel/RdbmsCentralRepoFactory.java | 191 +++++++++--------- 2 files changed, 96 insertions(+), 112 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java index fedf0b75c9..abbae1c867 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java @@ -3194,26 +3194,19 @@ abstract class RdbmsCentralRepo implements CentralRepository { // clear out the cache typeCache.invalidateAll(); - - Connection conn = connect(); - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; String sql = "SELECT * FROM correlation_types"; - try { - preparedStatement = conn.prepareStatement(sql); - resultSet = preparedStatement.executeQuery(); + try ( Connection conn = connect(); + PreparedStatement preparedStatement = conn.prepareStatement(sql); + ResultSet resultSet = preparedStatement.executeQuery();) { + while (resultSet.next()) { CorrelationAttributeInstance.Type aType = getCorrelationTypeFromResultSet(resultSet); typeCache.put(aType.getId(), aType); } } catch (SQLException ex) { throw new CentralRepoException("Error getting correlation types.", ex); // NON-NLS - } finally { - CentralRepoDbUtil.closeStatement(preparedStatement); - CentralRepoDbUtil.closeResultSet(resultSet); - CentralRepoDbUtil.closeConnection(conn); - } + } } /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java index f61fb42062..963809c5d7 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepoFactory.java @@ -81,7 +81,7 @@ public class RdbmsCentralRepoFactory { * connectionPool object directly. */ public boolean initializeDatabaseSchema() { - + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(selectedPlatform); String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); @@ -92,95 +92,88 @@ public class RdbmsCentralRepoFactory { // NOTE: the db_info table currenly only has 1 row, so having an index // provides no benefit. - Connection conn = null; - Statement stmt = null; - try { - conn = this.getEphemeralConnection(); + try (Connection conn = this.getEphemeralConnection();) { + if (null == conn) { LOGGER.log(Level.SEVERE, "Cannot initialize CR database, don't have a valid connection."); // NON-NLS return false; } - - stmt = conn.createStatement(); - // these setting PRAGMAs are SQLIte spcific - if (selectedPlatform == CentralRepoPlatforms.SQLITE) { - stmt.execute(PRAGMA_JOURNAL_WAL); - stmt.execute(PRAGMA_SYNC_OFF); - stmt.execute(PRAGMA_READ_UNCOMMITTED_TRUE); - stmt.execute(PRAGMA_ENCODING_UTF8); - stmt.execute(PRAGMA_PAGE_SIZE_4096); - stmt.execute(PRAGMA_FOREIGN_KEYS_ON); - } + try (Statement stmt = conn.createStatement();) { - // Create Organizations table - stmt.execute(getCreateOrganizationsTableStatement(selectedPlatform)); - - // Create Cases table and indexes - stmt.execute(getCreateCasesTableStatement(selectedPlatform)); - stmt.execute(getCasesOrgIdIndexStatement()); - stmt.execute(getCasesCaseUidIndexStatement()); - - stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); - stmt.execute(getAddDataSourcesNameIndexStatement()); - stmt.execute(getAddDataSourcesObjectIdIndexStatement()); - - stmt.execute(getCreateReferenceSetsTableStatement(selectedPlatform)); - stmt.execute(getReferenceSetsOrgIdIndexTemplate()); - - stmt.execute(getCreateCorrelationTypesTableStatement(selectedPlatform)); - - stmt.execute(getCreateDbInfoTableStatement(selectedPlatform)); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); - stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); - - // Create account_types and accounts tab;es which are referred by X_instances tables - stmt.execute(getCreateAccountTypesTableStatement(selectedPlatform)); - stmt.execute(getCreateAccountsTableStatement(selectedPlatform)); - - // Create a separate instance and reference table for each artifact type - List defaultCorrelationTypes = CorrelationAttributeInstance.getDefaultCorrelationTypes(); - - String reference_type_dbname; - String instance_type_dbname; - for (CorrelationAttributeInstance.Type type : defaultCorrelationTypes) { - reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); - instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); - - stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); - - // FUTURE: allow more than the FILES type - if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { - stmt.execute(String.format(getReferenceTypesTableTemplate(selectedPlatform), reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(getReferenceTypeValueIndexTemplate(), reference_type_dbname, reference_type_dbname)); - stmt.execute(String.format(getReferenceTypeValueKnownstatusIndexTemplate(), reference_type_dbname, reference_type_dbname)); + // these setting PRAGMAs are SQLIte spcific + if (selectedPlatform == CentralRepoPlatforms.SQLITE) { + stmt.execute(PRAGMA_JOURNAL_WAL); + stmt.execute(PRAGMA_SYNC_OFF); + stmt.execute(PRAGMA_READ_UNCOMMITTED_TRUE); + stmt.execute(PRAGMA_ENCODING_UTF8); + stmt.execute(PRAGMA_PAGE_SIZE_4096); + stmt.execute(PRAGMA_FOREIGN_KEYS_ON); } + + // Create Organizations table + stmt.execute(getCreateOrganizationsTableStatement(selectedPlatform)); + + // Create Cases table and indexes + stmt.execute(getCreateCasesTableStatement(selectedPlatform)); + stmt.execute(getCasesOrgIdIndexStatement()); + stmt.execute(getCasesCaseUidIndexStatement()); + + stmt.execute(getCreateDataSourcesTableStatement(selectedPlatform)); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); + + stmt.execute(getCreateReferenceSetsTableStatement(selectedPlatform)); + stmt.execute(getReferenceSetsOrgIdIndexTemplate()); + + stmt.execute(getCreateCorrelationTypesTableStatement(selectedPlatform)); + + stmt.execute(getCreateDbInfoTableStatement(selectedPlatform)); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMajor() + "')"); + stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + RdbmsCentralRepo.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + SOFTWARE_CR_DB_SCHEMA_VERSION.getMinor() + "')"); + + // Create account_types and accounts tab;es which are referred by X_instances tables + stmt.execute(getCreateAccountTypesTableStatement(selectedPlatform)); + stmt.execute(getCreateAccountsTableStatement(selectedPlatform)); + + // Create a separate instance and reference table for each artifact type + List defaultCorrelationTypes = CorrelationAttributeInstance.getDefaultCorrelationTypes(); + + String reference_type_dbname; + String instance_type_dbname; + for (CorrelationAttributeInstance.Type type : defaultCorrelationTypes) { + reference_type_dbname = CentralRepoDbUtil.correlationTypeToReferenceTableName(type); + instance_type_dbname = CentralRepoDbUtil.correlationTypeToInstanceTableName(type); + + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); + + // FUTURE: allow more than the FILES type + if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + stmt.execute(String.format(getReferenceTypesTableTemplate(selectedPlatform), reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(getReferenceTypeValueIndexTemplate(), reference_type_dbname, reference_type_dbname)); + stmt.execute(String.format(getReferenceTypeValueKnownstatusIndexTemplate(), reference_type_dbname, reference_type_dbname)); + } + } + createPersonaTables(stmt); + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS + return false; + } catch (CentralRepoException ex) { + LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS + return false; } - - createPersonaTables(stmt); } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "Error initializing db schema.", ex); // NON-NLS + LOGGER.log(Level.SEVERE, "Error connecting to database.", ex); // NON-NLS return false; - } catch (CentralRepoException ex) { - LOGGER.log(Level.SEVERE, "Error getting default correlation types. Likely due to one or more Type's with an invalid db table name."); // NON-NLS - return false; - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException ex2) { - LOGGER.log(Level.SEVERE, "Error closing statement.", ex2); - } - } - CentralRepoDbUtil.closeConnection(conn); } + return true; } @@ -190,17 +183,22 @@ public class RdbmsCentralRepoFactory { * @return True if success, False otherwise. */ public boolean insertDefaultDatabaseContent() { - Connection conn = this.getEphemeralConnection(); - if (null == conn) { + + boolean result; + try (Connection conn = this.getEphemeralConnection();) { + if (null == conn) { + return false; + } + + result = CentralRepoDbUtil.insertDefaultCorrelationTypes(conn) + && CentralRepoDbUtil.insertDefaultOrganization(conn) + && insertDefaultPersonaTablesContent(conn); + + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, String.format("Failed to populate default data in CR tables."), ex); return false; } - boolean result = CentralRepoDbUtil.insertDefaultCorrelationTypes(conn) - && CentralRepoDbUtil.insertDefaultOrganization(conn) - && insertDefaultPersonaTablesContent(conn); - - - CentralRepoDbUtil.closeConnection(conn); return result; } @@ -849,24 +847,17 @@ public class RdbmsCentralRepoFactory { } else if (accountType == Account.Type.PHONE) { typeId = CorrelationAttributeInstance.PHONE_TYPE_ID; } else { - ResultSet resultSet = null; - - PreparedStatement preparedStatementQuery = null; String querySql = "SELECT * FROM correlation_types WHERE display_name=?"; - try { - preparedStatementQuery = conn.prepareStatement(querySql); + try ( PreparedStatement preparedStatementQuery = conn.prepareStatement(querySql)) { preparedStatementQuery.setString(1, accountType.getDisplayName()); - - resultSet = preparedStatementQuery.executeQuery(); - if (resultSet.next()) { - typeId = resultSet.getInt("id"); + try (ResultSet resultSet = preparedStatementQuery.executeQuery();) { + if (resultSet.next()) { + typeId = resultSet.getInt("id"); + } } } catch (SQLException ex) { LOGGER.log(Level.SEVERE, String.format("Failed to get correlation typeId for account type %s.", accountType.getTypeName()), ex); - } finally { - CentralRepoDbUtil.closeStatement(preparedStatementQuery); - CentralRepoDbUtil.closeResultSet(resultSet); - } + } } return typeId;