diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java index 08bc0fa427..98f2855fcb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.casemodule; import java.util.List; +import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestStream; import org.sleuthkit.autopsy.ingest.IngestStreamClosedException; @@ -35,6 +36,11 @@ class DefaultIngestStream implements IngestStream { public void addFiles(List fileObjectIds) throws IngestStreamClosedException { // Do nothing } + + @Override + public IngestJob getIngestJob() { + throw new UnsupportedOperationException("DefaultIngestStream has no associated IngestJob"); + } @Override public synchronized boolean isClosed() { diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java index 9dd9a39fd4..207b83ed8a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java @@ -464,6 +464,42 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); } + + @Override + public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + this.deviceId = deviceId; + this.imagePath = dataSourcePath.toString(); + this.sectorSize = 0; + this.timeZone = Calendar.getInstance().getTimeZone().getID(); + this.ignoreFatOrphanFiles = false; + setDataSourceOptionsCalled = true; + + // Set up the data source before creating the ingest stream + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return null; + } + + // Now initialize the ingest stream + try { + ingestStream = IngestManager.getInstance().openIngestStream(image, settings); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error starting ingest modules", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return null; + } + + doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); + return ingestStream; + } /** * Sets the configuration of the data source processor without using the diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java index 957b887c75..55163535cb 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2019 Basis Technology Corp. + * Copyright 2018-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -97,68 +97,64 @@ import org.sleuthkit.datamodel.TskCoreException; * Image viewer part of the Media View layered pane. Uses JavaFX to display the * image. */ -@NbBundle.Messages({"MediaViewImagePanel.externalViewerButton.text=Open in External Viewer Ctrl+E", +@NbBundle.Messages({ + "MediaViewImagePanel.externalViewerButton.text=Open in External Viewer Ctrl+E", "MediaViewImagePanel.errorLabel.text=Could not load file into Media View.", - "MediaViewImagePanel.errorLabel.OOMText=Could not load file into Media View: insufficent memory."}) + "MediaViewImagePanel.errorLabel.OOMText=Could not load file into Media View: insufficent memory." +}) @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPanel { - private static final Image EXTERNAL = new Image(MediaViewImagePanel.class.getResource("/org/sleuthkit/autopsy/images/external.png").toExternalForm()); - private final static Logger LOGGER = Logger.getLogger(MediaViewImagePanel.class.getName()); - - private final boolean fxInited; - - private JFXPanel fxPanel; - private AbstractFile file; + private static final long serialVersionUID = 1L; + private static final Logger logger = Logger.getLogger(MediaViewImagePanel.class.getName()); + private static final double[] ZOOM_STEPS = { + 0.0625, 0.125, 0.25, 0.375, 0.5, 0.75, + 1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10}; + private static final double MIN_ZOOM_RATIO = 0.0625; // 6.25% + private static final double MAX_ZOOM_RATIO = 10.0; // 1000% + private static final Image externalImage = new Image(MediaViewImagePanel.class.getResource("/org/sleuthkit/autopsy/images/external.png").toExternalForm()); + private static final SortedSet supportedMimes = ImageUtils.getSupportedImageMimeTypes(); + private static final List supportedExtensions = ImageUtils.getSupportedImageExtensions().stream() + .map("."::concat) //NOI18N + .collect(Collectors.toList()); + + /* + * JFX components + */ + private final ProgressBar progressBar = new ProgressBar(); + private final MaskerPane maskerPane = new MaskerPane(); private Group masterGroup; private ImageTagsGroup tagsGroup; private ImageTagCreator imageTagCreator; private ImageView fxImageView; private ScrollPane scrollPane; - private final ProgressBar progressBar = new ProgressBar(); - private final MaskerPane maskerPane = new MaskerPane(); - + private Task readImageTask; + + /* + * Swing components + */ private final JPopupMenu imageTaggingOptions = new JPopupMenu(); private final JMenuItem createTagMenuItem; private final JMenuItem deleteTagMenuItem; private final JMenuItem hideTagsMenuItem; private final JMenuItem exportTagsMenuItem; - private final JFileChooser exportChooser; - private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); - + private JFXPanel fxPanel; + + /* + * State + */ + private final boolean fxInited; private double zoomRatio; private double rotation; // Can be 0, 90, 180, and 270. - - private boolean autoResize = true; // Auto resize when the user changes the size - // of the content viewer unless the user has used the zoom buttons. - private static final double[] ZOOM_STEPS = { - 0.0625, 0.125, 0.25, 0.375, 0.5, 0.75, - 1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10}; - - private static final double MIN_ZOOM_RATIO = 0.0625; // 6.25% - private static final double MAX_ZOOM_RATIO = 10.0; // 1000% + private boolean autoResize = true; // Auto resize when the user changes the size of the content viewer unless the user has used the zoom buttons. + private AbstractFile file; static { ImageIO.scanForPlugins(); } - /** - * mime types we should be able to display. if the mimetype is unknown we - * will fall back on extension and jpg/png header - */ - static private final SortedSet supportedMimes = ImageUtils.getSupportedImageMimeTypes(); - - /** - * extensions we should be able to display - */ - static private final List supportedExtensions = ImageUtils.getSupportedImageExtensions().stream() - .map("."::concat) //NOI18N - .collect(Collectors.toList()); - - private Task readImageTask; - /** * Creates new form MediaViewImagePanel */ @@ -168,7 +164,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan "MediaViewImagePanel.hideTagOption=Hide", "MediaViewImagePanel.exportTagOption=Export" }) - public MediaViewImagePanel() { + MediaViewImagePanel() { initComponents(); fxInited = org.sleuthkit.autopsy.core.Installer.isJavaFxInited(); @@ -354,14 +350,13 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan } private void showErrorNode(String errorMessage, AbstractFile file) { - final Button externalViewerButton = new Button(Bundle.MediaViewImagePanel_externalViewerButton_text(), new ImageView(EXTERNAL)); - externalViewerButton.setOnAction(actionEvent - -> //fx ActionEvent - /* - * TODO: why is the name passed into the action constructor? it - * means we duplicate this string all over the place -jm - */ new ExternalViewerAction(Bundle.MediaViewImagePanel_externalViewerButton_text(), new FileNode(file)) - .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) //Swing ActionEvent + final Button externalViewerButton = new Button(Bundle.MediaViewImagePanel_externalViewerButton_text(), new ImageView(externalImage)); + /* + * Tie a Swing action (ExternalViewerAction) to a JFX button action. + */ + externalViewerButton.setOnAction(actionEvent -> + new ExternalViewerAction(Bundle.MediaViewImagePanel_externalViewerButton_text(), new FileNode(file)) + .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) ); final VBox errorNode = new VBox(10, new Label(errorMessage), externalViewerButton); @@ -420,7 +415,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan "state", null, State.NONEMPTY)); } } catch (TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not retrieve image tags for file in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not retrieve image tags for file in case db", ex); //NON-NLS } scrollPane.setContent(masterGroup); } else { @@ -693,14 +688,14 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan private void rotateLeftButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_rotateLeftButtonActionPerformed autoResize = false; - + rotation = (rotation + 270) % 360; updateView(); }//GEN-LAST:event_rotateLeftButtonActionPerformed private void rotateRightButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_rotateRightButtonActionPerformed autoResize = false; - + rotation = (rotation + 90) % 360; updateView(); }//GEN-LAST:event_rotateRightButtonActionPerformed @@ -760,7 +755,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan Case.getCurrentCase().getServices().getTagsManager().deleteContentTag(contentViewerTag.getContentTag()); tagsGroup.getChildren().remove(tagInFocus); } catch (TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not delete image tag in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not delete image tag in case db", ex); //NON-NLS } scrollPane.setCursor(Cursor.DEFAULT); @@ -793,7 +788,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan ImageTag imageTag = buildImageTag(contentViewerTag); tagsGroup.getChildren().add(imageTag); } catch (TskCoreException | SerializationException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not save new image tag in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not save new image tag in case db", ex); //NON-NLS } scrollPane.setCursor(Cursor.DEFAULT); @@ -832,7 +827,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan ImageTagRegion newRegion = (ImageTagRegion) edit.getNewValue(); ContentViewerTagManager.updateTag(contentViewerTag, newRegion); } catch (SerializationException | TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.WARNING, "Could not save edit for image tag in case db", ex); //NON-NLS + logger.log(Level.WARNING, "Could not save edit for image tag in case db", ex); //NON-NLS } scrollPane.setCursor(Cursor.DEFAULT); }); @@ -916,7 +911,7 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan JOptionPane.showMessageDialog(null, Bundle.MediaViewImagePanel_successfulExport()); } catch (Exception ex) { //Runtime exceptions may spill out of ImageTagsUtil from JavaFX. //This ensures we (devs and users) have something when it doesn't work. - LOGGER.log(Level.WARNING, "Unable to export tagged image to disk", ex); //NON-NLS + logger.log(Level.WARNING, "Unable to export tagged image to disk", ex); //NON-NLS JOptionPane.showMessageDialog(null, Bundle.MediaViewImagePanel_unsuccessfulExport()); } return null; diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java index 20f3bb59bd..27ffec6d53 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java @@ -22,6 +22,8 @@ import java.nio.file.Path; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.ingest.IngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestStream; /** * Interface implemented by DataSourceProcessors in order to be supported by @@ -66,6 +68,31 @@ public interface AutoIngestDataSourceProcessor extends DataSourceProcessor { */ void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack); + + /** + * Adds a data source to the case database using a background task in a + * separate thread by calling DataSourceProcessor.run() method. Returns as + * soon as the background task is started. The background task uses a + * callback object to signal task completion and return results. Method can + * throw an exception for a system level problem. The exception should not + * be thrown for an issue related to bad input data. + * + * @param deviceId An ASCII-printable identifier for the device + * associated with the data source that is intended + * to be unique across multiple cases (e.g., a UUID). + * @param dataSourcePath Path to the data source. + * @param settings The ingest job settings. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callBack Callback that will be used by the background task + * to return results. + * + * @return The new ingest stream or null if an error occurred. Errors will be handled by the callback. + */ + default IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor"); + } + /** * A custom exception for the use of AutomatedIngestDataSourceProcessor. */ diff --git a/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java index a71876e88d..3617ce3670 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/GroupListPanel.java @@ -20,6 +20,8 @@ package org.sleuthkit.autopsy.discovery; import com.google.common.eventbus.Subscribe; import java.awt.Cursor; +import java.awt.Graphics2D; +import java.awt.font.FontRenderContext; import java.util.List; import java.util.Map; import javax.swing.DefaultListCellRenderer; @@ -204,11 +206,30 @@ final class GroupListPanel extends javax.swing.JPanel { if (newValue instanceof GroupKey) { String valueString = newValue.toString(); setToolTipText(valueString); - //if paths would be longer than 37 characters shorten them to be 37 characters - if (groupingAttribute instanceof FileSearch.ParentPathAttribute && valueString.length() > 37) { - valueString = valueString.substring(0, 16) + " ... " + valueString.substring(valueString.length() - 16); + + valueString += " (" + groupMap.get(newValue) + ")"; + + if (groupingAttribute instanceof FileSearch.ParentPathAttribute) { + // Using the list FontRenderContext instead of this because + // the label RenderContext was sometimes null, but this should work. + FontRenderContext context = ((Graphics2D) list.getGraphics()).getFontRenderContext(); + + //Determine the width of the string with the given font. + double stringWidth = getFont().getStringBounds(valueString, context).getWidth(); + // subtracting 10 from the width as a littl inset. + int listWidth = list.getWidth() - 10; + + if (stringWidth > listWidth) { + double avgCharWidth = Math.floor(stringWidth / valueString.length()); + + // The extra 5 is to account for the " ... " that is being added back. + int charToRemove = (int) Math.ceil((stringWidth - listWidth) / avgCharWidth) + 5; + int charactersToShow = (int) Math.ceil((valueString.length() - charToRemove) / 2); + valueString = valueString.substring(0, charactersToShow) + " ... " + valueString.substring(valueString.length() - charactersToShow); + } + } - newValue = valueString + " (" + groupMap.get(newValue) + ")"; + newValue = valueString; } super.getListCellRendererComponent(list, newValue, index, isSelected, cellHasFocus); return this; diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties index 6d3f5b4b66..77a2b221ee 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties @@ -36,3 +36,4 @@ MapPanel.zoomInBtn.text= MapPanel.zoomOutBtn.text= GeoFilterPanel.showLabel.text=Show: GeoFilterPanel.showLabel.toolTipText=Show: +GeoFilterPanel.atCBPanel.AccessibleContext.accessibleName= diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED index d16fa37761..ddec131b05 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED @@ -77,4 +77,5 @@ MapPanel.zoomInBtn.text= MapPanel.zoomOutBtn.text= GeoFilterPanel.showLabel.text=Show: GeoFilterPanel.showLabel.toolTipText=Show: +GeoFilterPanel.atCBPanel.AccessibleContext.accessibleName= WaypointExtractAction_label=Extract Files(s) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.form b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.form index f19020d8c6..720f4d8a55 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.form +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.form @@ -9,6 +9,14 @@ + + + + + + + + @@ -40,7 +48,7 @@ - + @@ -99,6 +107,9 @@ + + + @@ -148,7 +159,7 @@ - + @@ -190,5 +201,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java index c5d945dd31..df91b2963e 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.geolocation; import org.sleuthkit.autopsy.guiutils.CheckBoxListPanel; import java.awt.Color; import java.awt.Graphics; -import java.awt.GridBagConstraints; import java.awt.event.ActionListener; import java.awt.image.BufferedImage; import java.sql.ResultSet; @@ -82,43 +81,22 @@ class GeoFilterPanel extends javax.swing.JPanel { "GeoFilterPanel_DataSource_List_Title=Data Sources", "GeoFilterPanel_ArtifactType_List_Title=Types" }) + @SuppressWarnings("unchecked") GeoFilterPanel() { // numberModel is used in initComponents numberModel = new SpinnerNumberModel(10, 1, Integer.MAX_VALUE, 1); initComponents(); - // The gui builder cannot handle using CheckBoxListPanel due to its - // use of generics so we will initalize it here. - dsCheckboxPanel = new CheckBoxListPanel<>(); + dsCheckboxPanel = (CheckBoxListPanel)dsCBPanel; dsCheckboxPanel.setPanelTitle(Bundle.GeoFilterPanel_DataSource_List_Title()); dsCheckboxPanel.setPanelTitleIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/image.png"))); dsCheckboxPanel.setSetAllSelected(true); - atCheckboxPanel = new CheckBoxListPanel<>(); + atCheckboxPanel = (CheckBoxListPanel)atCBPanel; atCheckboxPanel.setPanelTitle(Bundle.GeoFilterPanel_ArtifactType_List_Title()); atCheckboxPanel.setPanelTitleIcon(new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/extracted_content.png"))); atCheckboxPanel.setSetAllSelected(true); - - GridBagConstraints gridBagConstraints = new GridBagConstraints(); - gridBagConstraints.gridx = 0; - gridBagConstraints.gridy = 3; - gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; - gridBagConstraints.weightx = 1.0; - gridBagConstraints.weighty = 1.0; - gridBagConstraints.insets = new java.awt.Insets(0, 15, 0, 15); - add(dsCheckboxPanel, gridBagConstraints); - - gridBagConstraints = new GridBagConstraints(); - gridBagConstraints.gridx = 0; - gridBagConstraints.gridy = 4; - gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; - gridBagConstraints.weightx = 1.0; - gridBagConstraints.weighty = 1.0; - gridBagConstraints.insets = new java.awt.Insets(0, 15, 0, 15); - add(atCheckboxPanel, gridBagConstraints); } @Override @@ -221,7 +199,11 @@ class GeoFilterPanel extends javax.swing.JPanel { javax.swing.JPanel buttonPanel = new javax.swing.JPanel(); applyButton = new javax.swing.JButton(); javax.swing.JLabel optionsLabel = new javax.swing.JLabel(); + dsCBPanel = new CheckBoxListPanel(); + atCBPanel = new CheckBoxListPanel(); + setMinimumSize(new java.awt.Dimension(10, 700)); + setPreferredSize(new java.awt.Dimension(300, 700)); setLayout(new java.awt.GridBagLayout()); waypointSettings.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(GeoFilterPanel.class, "GeoFilterPanel.waypointSettings.border.title"))); // NOI18N @@ -268,6 +250,7 @@ class GeoFilterPanel extends javax.swing.JPanel { waypointSettings.add(showWaypointsWOTSCheckBox, gridBagConstraints); daysSpinner.setEnabled(false); + daysSpinner.setMaximumSize(new java.awt.Dimension(100, 26)); daysSpinner.setPreferredSize(new java.awt.Dimension(75, 26)); gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 2; @@ -298,7 +281,7 @@ class GeoFilterPanel extends javax.swing.JPanel { gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; - gridBagConstraints.insets = new java.awt.Insets(5, 15, 9, 15); + gridBagConstraints.insets = new java.awt.Insets(5, 15, 9, 25); add(waypointSettings, gridBagConstraints); buttonPanel.setLayout(new java.awt.GridBagLayout()); @@ -316,7 +299,7 @@ class GeoFilterPanel extends javax.swing.JPanel { gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; gridBagConstraints.weightx = 1.0; - gridBagConstraints.insets = new java.awt.Insets(9, 15, 0, 15); + gridBagConstraints.insets = new java.awt.Insets(9, 15, 0, 25); add(buttonPanel, gridBagConstraints); optionsLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/blueGeo16.png"))); // NOI18N @@ -327,6 +310,29 @@ class GeoFilterPanel extends javax.swing.JPanel { gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; gridBagConstraints.insets = new java.awt.Insets(0, 15, 0, 0); add(optionsLabel, gridBagConstraints); + + dsCBPanel.setMinimumSize(new java.awt.Dimension(150, 250)); + dsCBPanel.setPreferredSize(new java.awt.Dimension(150, 250)); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 3; + gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(5, 15, 9, 25); + add(dsCBPanel, gridBagConstraints); + + atCBPanel.setMinimumSize(new java.awt.Dimension(150, 250)); + atCBPanel.setPreferredSize(new java.awt.Dimension(150, 250)); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 4; + gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(5, 15, 9, 25); + add(atCBPanel, gridBagConstraints); + atCBPanel.getAccessibleContext().setAccessibleName(org.openide.util.NbBundle.getMessage(GeoFilterPanel.class, "GeoFilterPanel.atCBPanel.AccessibleContext.accessibleName")); // NOI18N }// //GEN-END:initComponents private void allButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_allButtonActionPerformed @@ -341,8 +347,10 @@ class GeoFilterPanel extends javax.swing.JPanel { // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JRadioButton allButton; private javax.swing.JButton applyButton; + private javax.swing.JPanel atCBPanel; private javax.swing.JLabel daysLabel; private javax.swing.JSpinner daysSpinner; + private javax.swing.JPanel dsCBPanel; private javax.swing.JRadioButton mostRecentButton; private javax.swing.JLabel showLabel; private javax.swing.JCheckBox showWaypointsWOTSCheckBox; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java index 4e8e9c4019..a2687d5c1d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java @@ -58,6 +58,11 @@ class IngestJobInputStream implements IngestStream { } ingestJob.addStreamingIngestFiles(fileObjectIds); } + + @Override + public IngestJob getIngestJob() { + return ingestJob; + } @Override public synchronized void close() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java index 62a42af208..77001531be 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java @@ -34,6 +34,13 @@ public interface IngestStream { * @throws IngestStreamClosedException */ void addFiles(List fileObjectIds) throws IngestStreamClosedException; + + /** + * Get the ingest job associated with this ingest stream. + * + * @return The IngestJob. + */ + IngestJob getIngestJob(); /** * Closes the ingest stream. Should be called after all files from data diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java index 708ce7ee14..5bc3f46fac 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java @@ -275,6 +275,21 @@ final class AutoIngestJobLogger { void logIngestJobSettingsErrors() throws AutoIngestJobLoggerException, InterruptedException { log(MessageCategory.ERROR, "Failed to analyze data source due to settings errors"); } + + /** + * Logs failure to analyze a data source, possibly due to ingest job settings errors. + * Used with streaming ingest since incorrect settings are the most likely cause + * of the error. + * + * @throws AutoIngestJobLoggerException if there is an error writing the log + * message. + * @throws InterruptedException if interrupted while blocked waiting + * to acquire an exclusive lock on the + * log file. + */ + void logProbableIngestJobSettingsErrors() throws AutoIngestJobLoggerException, InterruptedException { + log(MessageCategory.ERROR, "Failed to analyze data source, probably due to ingest settings errors"); + } /** * Logs failure to analyze a data source due to ingest module startup diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 48b86025af..7c817e7739 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -101,6 +101,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; +import org.sleuthkit.autopsy.ingest.IngestStream; import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException; import org.sleuthkit.autopsy.keywordsearch.Server; import org.sleuthkit.datamodel.Content; @@ -165,6 +166,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen private AutoIngestJob currentJob; @GuardedBy("jobsLock") private List completedJobs; + private IngestStream currentIngestStream = null; private CoordinationService coordinationService; private JobProcessingTask jobProcessingTask; private Future jobProcessingTaskFuture; @@ -2443,6 +2445,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen return; } + currentIngestStream = null; runDataSourceProcessor(caseForJob, dataSource); if (dataSource.getContent().isEmpty()) { currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now())); @@ -2558,7 +2561,29 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen caseForJob.notifyAddingDataSource(taskId); jobLogger.logDataSourceProcessorSelected(selectedProcessor.getDataSourceType()); sysLogger.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()}); - selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + if (selectedProcessor.supportsIngestStream()) { + IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); + if (! ingestJobSettings.getWarnings().isEmpty()) { + for (String warning : ingestJobSettings.getWarnings()) { + sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); + } + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logIngestJobSettingsErrors(); + throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Error(s) in ingest job settings for " + manifestPath); + } + currentIngestStream = selectedProcessor.processWithIngestStream(dataSource.getDeviceId(), dataSource.getPath(), ingestJobSettings, progressMonitor, callBack); + if (currentIngestStream == null) { + // Either there was a failure to add the data source object to the database or the ingest settings were bad. + // An error in the ingest settings is the more likely scenario. + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logProbableIngestJobSettingsErrors(); + throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Error initializing processing for " + manifestPath + ", probably due to an ingest settings error"); + } + } else { + selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + } ingestLock.wait(); // at this point we got the content object(s) from the current DSP. @@ -2568,6 +2593,12 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen // move onto the the next DSP that can process this data source jobLogger.logDataSourceProcessorError(selectedProcessor.getDataSourceType()); logDataSourceProcessorResult(dataSource); + + // If we had created an ingest stream, close it + if (currentIngestStream != null) { + currentIngestStream.stop(); + currentIngestStream = null; + } continue; } @@ -2674,69 +2705,77 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { - IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); - List settingsWarnings = ingestJobSettings.getWarnings(); - if (settingsWarnings.isEmpty()) { - IngestJobStartResult ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); - IngestJob ingestJob = ingestJobStartResult.getJob(); - if (null != ingestJob) { - currentJob.setIngestJob(ingestJob); - /* - * Block until notified by the ingest job event - * listener or until interrupted because auto ingest - * is shutting down. - */ - ingestLock.wait(); - sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); - IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); - for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { - AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); - if (!snapshot.isCancelled()) { - List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); - if (!cancelledModules.isEmpty()) { - sysLogger.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath)); - currentJob.setErrorsOccurred(true); - setErrorsOccurredFlagForCase(caseDirectoryPath); - for (String module : snapshot.getCancelledDataSourceIngestModules()) { - sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); - nestedJobLogger.logIngestModuleCancelled(module); - } - } - nestedJobLogger.logAnalysisCompleted(); - } else { - currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); + IngestJob ingestJob; + IngestJobStartResult ingestJobStartResult = null; + if (currentIngestStream == null) { + IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); + List settingsWarnings = ingestJobSettings.getWarnings(); + if (! settingsWarnings.isEmpty()) { + for (String warning : settingsWarnings) { + sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); + } + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logIngestJobSettingsErrors(); + throw new AnalysisStartupException("Error(s) in ingest job settings"); + } + + + ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); + ingestJob = ingestJobStartResult.getJob(); + } else { + ingestJob = currentIngestStream.getIngestJob(); + } + + if (null != ingestJob) { + currentJob.setIngestJob(ingestJob); + /* + * Block until notified by the ingest job event + * listener or until interrupted because auto ingest + * is shutting down. + */ + ingestLock.wait(); + sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); + IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); + for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { + AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); + if (!snapshot.isCancelled()) { + List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); + if (!cancelledModules.isEmpty()) { + sysLogger.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath)); currentJob.setErrorsOccurred(true); setErrorsOccurredFlagForCase(caseDirectoryPath); - nestedJobLogger.logAnalysisCancelled(); - CancellationReason cancellationReason = snapshot.getCancellationReason(); - if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { - throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); + for (String module : snapshot.getCancelledDataSourceIngestModules()) { + sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); + nestedJobLogger.logIngestModuleCancelled(module); } } + nestedJobLogger.logAnalysisCompleted(); + } else { + currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + nestedJobLogger.logAnalysisCancelled(); + CancellationReason cancellationReason = snapshot.getCancellationReason(); + if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { + throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); + } } - } else if (!ingestJobStartResult.getModuleErrors().isEmpty()) { - for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) { - sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); - } - currentJob.setErrorsOccurred(true); - setErrorsOccurredFlagForCase(caseDirectoryPath); - jobLogger.logIngestModuleStartupErrors(); - throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); - } else { - sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); - currentJob.setErrorsOccurred(true); - setErrorsOccurredFlagForCase(caseDirectoryPath); - jobLogger.logAnalysisStartupError(); - throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException()); } - } else { - for (String warning : settingsWarnings) { - sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); + } else if (ingestJobStartResult != null && !ingestJobStartResult.getModuleErrors().isEmpty()) { + for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) { + sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); } currentJob.setErrorsOccurred(true); setErrorsOccurredFlagForCase(caseDirectoryPath); - jobLogger.logIngestJobSettingsErrors(); - throw new AnalysisStartupException("Error(s) in ingest job settings"); + jobLogger.logIngestModuleStartupErrors(); + throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); + } else if (ingestJobStartResult != null) { + sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); + currentJob.setErrorsOccurred(true); + setErrorsOccurredFlagForCase(caseDirectoryPath); + jobLogger.logAnalysisStartupError(); + throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException()); } } } finally { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 774f6ea3e5..34cc943e34 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -365,7 +365,7 @@ public class GroupManager { } else { //group == null // It may be that this was the last unanalyzed file in the group, so test // whether the group is now fully analyzed. - return popuplateIfAnalyzed(groupKey, null); + return populateIfAnalyzed(groupKey, null); } } @@ -574,7 +574,7 @@ public class GroupManager { * 'populateIfAnalyzed' will still not return a group and therefore * this method will never mark the group as unseen. */ - group = popuplateIfAnalyzed(groupKey, null); + group = populateIfAnalyzed(groupKey, null); } else { //if there is aleady a group that was previously deemed fully analyzed, then add this newly analyzed file to it. group.addFile(fileID); @@ -680,7 +680,7 @@ public class GroupManager { } else if (groupKey.getValue().toString().equalsIgnoreCase(this.currentPathGroup.getValue().toString()) == false) { // mark the last path group as analyzed getDrawableDB().markGroupAnalyzed(currentPathGroup); - popuplateIfAnalyzed(currentPathGroup, null); + populateIfAnalyzed(currentPathGroup, null); currentPathGroup = groupKey; } @@ -698,7 +698,7 @@ public class GroupManager { try { if (currentPathGroup != null) { getDrawableDB().markGroupAnalyzed(currentPathGroup); - popuplateIfAnalyzed(currentPathGroup, null); + populateIfAnalyzed(currentPathGroup, null); currentPathGroup = null; } } catch (TskCoreException ex) { @@ -713,7 +713,7 @@ public class GroupManager { * * @returns null if Group is not ready to be viewed */ - synchronized private DrawableGroup popuplateIfAnalyzed(GroupKey groupKey, ReGroupTask task) { + synchronized private DrawableGroup populateIfAnalyzed(GroupKey groupKey, ReGroupTask task) { /* * If this method call is part of a ReGroupTask and that task is * cancelled, no-op. @@ -735,7 +735,7 @@ public class GroupManager { if (groupKey.getAttribute() != DrawableAttribute.PATH || getDrawableDB().isGroupAnalyzed(groupKey)) { Set fileIDs = getFileIDsInGroup(groupKey); - if (Objects.nonNull(fileIDs)) { + if (Objects.nonNull(fileIDs) && ! fileIDs.isEmpty()) { long examinerID = collaborativeModeProp.get() ? -1 : controller.getCaseDatabase().getCurrentExaminer().getId(); final boolean groupSeen = getDrawableDB().isGroupSeenByExaminer(groupKey, examinerID); @@ -866,7 +866,7 @@ public class GroupManager { p++; updateMessage(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString()) + valForDataSource.getValue()); updateProgress(p, valsByDataSource.size()); - popuplateIfAnalyzed(new GroupKey<>(groupBy, valForDataSource.getValue(), valForDataSource.getKey()), this); + populateIfAnalyzed(new GroupKey<>(groupBy, valForDataSource.getValue(), valForDataSource.getKey()), this); } Optional viewedGroup diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml index bbc4d12f61..4cedc25d2b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SEUQAMappings.xml @@ -12,7 +12,7 @@ Each splitToken contains a single mapping of a raw URL substring to its regex eq SearchEngine: engine: The engines basic name - domainSubstring: The domain of the URL such that it can uniquely be identified as given engine. + domainSubstring: The domain of the URL such that it can be identified as given engine. Should not have leading or trailing '.' splitToken: plainToken: The string in the URL that is immediately followed by the actual query. @@ -25,30 +25,30 @@ splitToken: --> - + - + - + - + - + - + - + @@ -59,28 +59,28 @@ splitToken: - + - + - + - + - + @@ -92,22 +92,22 @@ splitToken: - + - + - + - + - + @@ -116,28 +116,28 @@ splitToken: - + - + - + - + - + - + - + - + diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index eeb6e5a987..0dde950a05 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -22,10 +22,15 @@ import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.util.Arrays; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -54,7 +59,7 @@ import org.xml.sax.SAXException; * artifacts, and extracting search text from them. * * - * To add search engines, edit SearchEngines.xml under RecentActivity + * To add search engines, edit SEUQAMappings.xml under RecentActivity * */ @NbBundle.Messages({ @@ -107,11 +112,13 @@ class SearchEngineURLQueryAnalyzer extends Extract { private final String engineName; private final String domainSubstring; private final List keyPairs; + private final Pattern domainRegexPattern; private int count; SearchEngine(String engineName, String domainSubstring, List keyPairs) { this.engineName = engineName; this.domainSubstring = domainSubstring; + domainRegexPattern = Pattern.compile("^(.*[./])?" + domainSubstring + "([./].*)?$"); this.keyPairs = keyPairs; count = 0; } @@ -127,6 +134,10 @@ class SearchEngineURLQueryAnalyzer extends Extract { String getDomainSubstring() { return domainSubstring; } + + Pattern getDomainRegexPattern() { + return domainRegexPattern; + } int getTotal() { return count; @@ -202,20 +213,21 @@ class SearchEngineURLQueryAnalyzer extends Extract { * * @param domain domain as part of the URL * - * @return supported search engine the domain belongs to or null if no match - * is found + * @return supported search engine(s) the domain belongs to (list may be empty) * */ - private static SearchEngineURLQueryAnalyzer.SearchEngine getSearchEngineFromUrl(String domain) { + private static Collection getSearchEngineFromUrl(String domain) { + List supportedEngines = new ArrayList<>(); if (engines == null) { - return null; + return supportedEngines; } for (SearchEngine engine : engines) { - if (domain.contains(engine.getDomainSubstring())) { - return engine; + Matcher matcher = engine.getDomainRegexPattern().matcher(domain); + if (matcher.matches()) { + supportedEngines.add(engine); } } - return null; + return supportedEngines; } /** @@ -294,8 +306,9 @@ class SearchEngineURLQueryAnalyzer extends Extract { int totalQueries = 0; try { //from blackboard_artifacts - Collection listArtifacts = currentCase.getSleuthkitCase().getMatchingArtifacts("WHERE (blackboard_artifacts.artifact_type_id = '" + ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID() //NON-NLS - + "' OR blackboard_artifacts.artifact_type_id = '" + ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID() + "') "); //List of every 'web_history' and 'bookmark' artifact NON-NLS + Collection listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts( + Arrays.asList(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_BOOKMARK), new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY)), + Arrays.asList(dataSource.getId())); logger.log(Level.INFO, "Processing {0} blackboard artifacts.", listArtifacts.size()); //NON-NLS for (BlackboardArtifact artifact : listArtifacts) { @@ -304,51 +317,54 @@ class SearchEngineURLQueryAnalyzer extends Extract { } //initializing default attributes - String query = ""; String searchEngineDomain = ""; String browser = ""; long last_accessed = -1; - long fileId = artifact.getObjectID(); - boolean isFromSource = tskCase.isFileFromSource(dataSource, fileId); - if (!isFromSource) { - //File was from a different dataSource. Skipping. - continue; - } - - AbstractFile file = tskCase.getAbstractFileById(fileId); + AbstractFile file = tskCase.getAbstractFileById(artifact.getObjectID()); if (file == null) { continue; } - SearchEngineURLQueryAnalyzer.SearchEngine se = null; - //from blackboard_attributes - Collection listAttributes = currentCase.getSleuthkitCase().getMatchingAttributes("WHERE artifact_id = " + artifact.getArtifactID()); //NON-NLS - - for (BlackboardAttribute attribute : listAttributes) { - if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL.getTypeID()) { - final String urlString = attribute.getValueString(); - se = getSearchEngineFromUrl(urlString); - if (se == null) { - break; - } - - query = extractSearchEngineQuery(se, attribute.getValueString()); - if (query.equals("")) //False positive match, artifact was not a query. NON-NLS - { - break; - } - - } else if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID()) { - browser = attribute.getValueString(); - } else if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID()) { - searchEngineDomain = attribute.getValueString(); - } else if (attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID()) { - last_accessed = attribute.getValueLong(); + // Try search engines on the URL to see if any produce a search string + Set searchQueries = new HashSet<>(); + BlackboardAttribute urlAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL)); + if (urlAttr == null) { + continue; + } + + final String urlString = urlAttr.getValueString(); + Collection possibleSearchEngines = getSearchEngineFromUrl(urlString); + for (SearchEngineURLQueryAnalyzer.SearchEngine se : possibleSearchEngines) { + String query = extractSearchEngineQuery(se, urlString); + // If we have a non-empty query string, add it to the list + if ( !query.equals("")) { + searchQueries.add(query); + se.increment(); } } + + // If we didn't extract any search queries, go on to the next artifact + if (searchQueries.isEmpty()) { + continue; + } + + // Extract the rest of the fields needed for the web search artifact + BlackboardAttribute browserAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME)); + if (browserAttr != null) { + browser = browserAttr.getValueString(); + } + BlackboardAttribute domainAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN)); + if (domainAttr != null) { + searchEngineDomain = domainAttr.getValueString(); + } + BlackboardAttribute lastAccessAttr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED)); + if (lastAccessAttr != null) { + last_accessed = lastAccessAttr.getValueLong(); + } - if (se != null && !query.equals("")) { //NON-NLS + // Make an artifact for each distinct query + for (String query : searchQueries) { // If date doesn't exist, change to 0 (instead of 1969) if (last_accessed == -1) { last_accessed = 0; @@ -367,7 +383,6 @@ class SearchEngineURLQueryAnalyzer extends Extract { NbBundle.getMessage(this.getClass(), "SearchEngineURLQueryAnalyzer.parentModuleName"), last_accessed)); postArtifact(createArtifactWithAttributes(ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY, file, bbattributes)); - se.increment(); ++totalQueries; } }