codex-admin/autopsy
1
0
Fork 0
mirror of https://github.com/elisspace/autopsy.git synced 2026-09-04 06:50:00 +00:00
Code Issues Releases Wiki Activity

On bulk inserts, disregard constraint violations due to duplicates

Browse Source
This commit is contained in:
Maxwell Koo
2017-06-28 08:21:44 -04:00
parent 531434660c
commit dfdb00a55e
5 changed files with 87 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java
View File

@@ -32,7 +32,6 @@ import java.sql.Types;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
@@ -100,7 +99,6 @@ public abstract class AbstractSqlEamDb implements EamDb {
// }
// // else, schema is current
// }
/**
* Setup and create a connection to the selected database implementation
*/
@@ -550,8 +548,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
}
/**
* Retrieves eamArtifact instances from the database that are associated with
* the eamArtifactType and eamArtifactValue of the given eamArtifact.
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValue of the given eamArtifact.
*
* @param eamArtifact The type/value to look up (artifact with 0 instances)
*
@@ -599,8 +597,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
}
/**
* Retrieves eamArtifact instances from the database that are associated with
* the aType and filePath
* Retrieves eamArtifact instances from the database that are associated
* with the aType and filePath
*
* @param aType EamArtifact.Type to search for
* @param filePath File path to search for
@@ -655,7 +653,7 @@ public abstract class AbstractSqlEamDb implements EamDb {
* associated with the ArtifactType and artifactValue of the given artifact.
*
* @param eamArtifact Artifact with artifactType and artifactValue to search
* for
* for
*
* @return Number of artifact instances having ArtifactType and
* ArtifactValue.
@@ -698,7 +696,7 @@ public abstract class AbstractSqlEamDb implements EamDb {
* case_id/datasource_id tuples in the database) expressed as a percentage.
*
* @param eamArtifact Artifact with artifactType and artifactValue to search
* for
* for
*
* @return Int between 0 and 100
*/
@@ -716,7 +714,7 @@ public abstract class AbstractSqlEamDb implements EamDb {
* the given artifact.
*
* @param eamArtifact Artifact with artifactType and artifactValue to search
* for
* for
*
* @return Number of unique tuples
*/
@@ -806,7 +804,7 @@ public abstract class AbstractSqlEamDb implements EamDb {
* @param eamInstance Instance with caseName and dataSource to search for
*
* @param eamInstance Instance with caseDisplayName and dataSource to search
* for
* for
*
* @return Number of artifact instances having caseDisplayName and
* dataSource
@@ -874,6 +872,13 @@ public abstract class AbstractSqlEamDb implements EamDb {
}
}
/**
* Get the conflict clause for bulk update statements
*
* @return The conflict clause for bulk update statements
*/
protected abstract String getConflictClause();
/**
* Executes a bulk insert of the eamArtifacts added from the
* prepareBulkArtifact() method
@@ -899,7 +904,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
sql.append(tableName);
sql.append(" (case_id, data_source_id, value, file_path, known_status, comment) ");
sql.append("VALUES ((SELECT id FROM cases WHERE case_uid=? LIMIT 1), ");
sql.append("(SELECT id FROM data_sources WHERE device_id=? LIMIT 1), ?, ?, ?, ?)");
sql.append("(SELECT id FROM data_sources WHERE device_id=? LIMIT 1), ?, ?, ?, ?) ");
sql.append(getConflictClause());
bulkPs = conn.prepareStatement(sql.toString());
@@ -949,7 +955,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
try {
String sql = "INSERT INTO cases(case_uid, org_id, case_name, creation_date, case_number, "
+ "examiner_name, examiner_email, examiner_phone, notes) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) "
+ getConflictClause();
bulkPs = conn.prepareStatement(sql);
for (EamCase eamCase : cases) {
@@ -987,8 +994,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
}
/**
* Sets an eamArtifact instance as knownStatus = "Bad". If eamArtifact exists,
* it is updated. If eamArtifact does not exist nothing happens
* Sets an eamArtifact instance as knownStatus = "Bad". If eamArtifact
* exists, it is updated. If eamArtifact does not exist nothing happens
*
* @param eamArtifact Artifact containing exactly one (1) ArtifactInstance.
*/
@@ -1055,7 +1062,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
}
/**
* Gets list of matching eamArtifact instances that have knownStatus = "Bad".
* Gets list of matching eamArtifact instances that have knownStatus =
* "Bad".
*
* @param eamArtifact Artifact containing Type and Value
*
@@ -1442,9 +1450,10 @@ public abstract class AbstractSqlEamDb implements EamDb {
/**
* Add a new global file instance to the bulk collection
*
*
* @param eamGlobalFileInstance The global file instance to add
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public void prepareGlobalFileInstance(EamGlobalFileInstance eamGlobalFileInstance) throws EamDbException {
@@ -1460,8 +1469,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
/**
* Insert the bulk collection of Global File Instances
*
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public void bulkInsertGlobalFileInstances() throws EamDbException {
@@ -1476,7 +1485,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
PreparedStatement bulkPs = null;
try {
for (EamArtifact.Type type : artifactTypes) {
String sql = "INSERT INTO global_files(global_reference_set_id, value, known_status, comment) VALUES (?, ?, ?, ?)";
String sql = "INSERT INTO global_files(global_reference_set_id, value, known_status, comment) VALUES (?, ?, ?, ?) "
+ getConflictClause();
bulkPs = conn.prepareStatement(sql);
@@ -1507,10 +1517,12 @@ public abstract class AbstractSqlEamDb implements EamDb {
/**
* Get all global file instances having a given MD5 hash
*
*
* @param MD5Hash The hash to lookup
*
* @return List of all global file instances with a given hash
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public List<EamGlobalFileInstance> getGlobalFileInstancesByHash(String MD5Hash) throws EamDbException {
@@ -1543,7 +1555,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
* Add a new EamArtifact.Type to the db.
*
* @param newType New type to add.
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public void newCorrelationArtifactType(EamArtifact.Type newType) throws EamDbException {
@@ -1575,7 +1588,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
*
* @return List of EamArtifact.Type's. If none are defined in the database,
* the default list will be returned.
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public List<EamArtifact.Type> getCorrelationArtifactTypes() throws EamDbException {
@@ -1609,7 +1623,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
*
* @return List of enabled EamArtifact.Type's. If none are defined in the
* database, the default list will be returned.
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public List<EamArtifact.Type> getEnabledCorrelationArtifactTypes() throws EamDbException {
@@ -1643,7 +1658,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
*
* @return List of supported EamArtifact.Type's. If none are defined in the
* database, the default list will be returned.
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public List<EamArtifact.Type> getSupportedCorrelationArtifactTypes() throws EamDbException {
@@ -1675,7 +1691,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
* Update a EamArtifact.Type.
*
* @param aType EamArtifact.Type to update.
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public void updateCorrelationArtifactType(EamArtifact.Type aType) throws EamDbException {
@@ -1707,7 +1724,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
* @param typeName Name of Type to get
*
* @return EamArtifact.Type or null if it doesn't exist.
* @throws EamDbException
*
* @throws EamDbException
*/
@Override
public EamArtifact.Type getCorrelationArtifactTypeByName(String typeName) throws EamDbException {

7
CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java
View File

@@ -33,6 +33,8 @@ public class PostgresEamDb extends AbstractSqlEamDb {
private final static Logger LOGGER = Logger.getLogger(PostgresEamDb.class.getName());
private final static String CONFLICT_CLAUSE = "ON CONFLICT DO NOTHING";
private static PostgresEamDb instance;
private static final int CONN_POOL_SIZE = 10;
@@ -162,6 +164,11 @@ public class PostgresEamDb extends AbstractSqlEamDb {
}
}
@Override
protected String getConflictClause() {
return CONFLICT_CLAUSE;
}
@Override
public List<String> getBadTags() {
return dbSettings.getBadTags();

41
CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java
View File

@@ -36,8 +36,7 @@ import org.sleuthkit.autopsy.coreutils.TextConverter;
import org.sleuthkit.autopsy.coreutils.TextConverterException;
/**
* Settings for the Postgres implementation of the Central Repository
* database
* Settings for the Postgres implementation of the Central Repository database
*/
public final class PostgresEamDbSettings {
@@ -148,8 +147,8 @@ public final class PostgresEamDbSettings {
* Get the full connection URL as a String
*
* @param usePostgresDb Connect to the 'postgres' database when testing
* connectivity and creating the main database.
*
* connectivity and creating the main database.
*
* @return
*/
public String getConnectionURL(boolean usePostgresDb) {
@@ -167,8 +166,9 @@ public final class PostgresEamDbSettings {
}
/**
* Use the current settings to get an ephemeral client connection for testing.
*
* Use the current settings to get an ephemeral client connection for
* testing.
*
* @return Connection or null.
*/
private Connection getEphemeralConnection(boolean usePostgresDb) {
@@ -191,9 +191,9 @@ public final class PostgresEamDbSettings {
}
/**
* Use the current settings and the validation query
* to test the connection to the database.
*
* Use the current settings and the validation query to test the connection
* to the database.
*
* @return true if successfull connection, else false.
*/
public boolean verifyConnection() {
@@ -201,7 +201,7 @@ public final class PostgresEamDbSettings {
if (null == conn) {
return false;
}
boolean result = EamDbUtil.executeValidationQuery(conn, VALIDATION_QUERY);
EamDbUtil.closeConnection(conn);
return result;
@@ -209,7 +209,7 @@ public final class PostgresEamDbSettings {
/**
* Check to see if the database exists.
*
*
* @return true if exists, else false
*/
public boolean verifyDatabaseExists() {
@@ -238,11 +238,11 @@ public final class PostgresEamDbSettings {
}
return false;
}
/**
* Use the current settings and the schema version query
* to test the database schema.
*
* Use the current settings and the schema version query to test the
* database schema.
*
* @return true if successfull connection, else false.
*/
public boolean verifyDatabaseSchema() {
@@ -275,8 +275,9 @@ public final class PostgresEamDbSettings {
EamDbUtil.closeConnection(conn);
}
return true;
}
/**
* Initialize the database schema.
*
@@ -305,7 +306,6 @@ public final class PostgresEamDbSettings {
// NOTE: The organizations will only have a small number of rows, so
// an index is probably not worthwhile.
StringBuilder createCasesTable = new StringBuilder();
createCasesTable.append("CREATE TABLE IF NOT EXISTS cases (");
createCasesTable.append("id SERIAL PRIMARY KEY,");
@@ -402,7 +402,6 @@ public final class PostgresEamDbSettings {
// NOTE: the db_info table currenly only has 1 row, so having an index
// provides no benefit.
Connection conn = null;
try {
conn = getEphemeralConnection(false);
@@ -514,9 +513,9 @@ public final class PostgresEamDbSettings {
}
/**
* To prevent issues where one command can honor case and another cannot,
* we will force the dbname to lower case.
*
* To prevent issues where one command can honor case and another cannot, we
* will force the dbname to lower case.
*
* @return the dbName
*/
public String getDbName() {

7
CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java
View File

@@ -25,7 +25,6 @@ import java.sql.Statement;
import java.util.List;
import java.util.logging.Level;
import org.apache.commons.dbcp2.BasicDataSource;
import org.openide.util.Exceptions;
import org.sleuthkit.autopsy.coreutils.Logger;
/**
@@ -162,6 +161,12 @@ public class SqliteEamDb extends AbstractSqlEamDb {
}
}
@Override
protected String getConflictClause() {
// For sqlite, our conflict clause is part of the table schema
return "";
}
@Override
public List<String> getBadTags() {
return dbSettings.getBadTags();

16
CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java
View File

@@ -256,8 +256,8 @@ public final class SqliteEamDbSettings {
createCasesTable.append("examiner_email text NOT NULL,");
createCasesTable.append("examiner_phone text NOT NULL,");
createCasesTable.append("notes text NOT NULL,");
createCasesTable.append("foreign key (org_id) references organizations(id) on update set null on delete set null,");
createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid)");
createCasesTable.append("CONSTRAINT case_uid_unique UNIQUE(case_uid) ON CONFLICT IGNORE,");
createCasesTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL");
createCasesTable.append(")");
// NOTE: when there are few cases in the cases table, these indices may not be worthwhile
@@ -281,7 +281,7 @@ public final class SqliteEamDbSettings {
createGlobalReferenceSetsTable.append("set_name text NOT NULL,");
createGlobalReferenceSetsTable.append("version text NOT NULL,");
createGlobalReferenceSetsTable.append("import_date text NOT NULL,");
createGlobalReferenceSetsTable.append("foreign key (org_id) references organizations(id) on update set null on delete set null");
createGlobalReferenceSetsTable.append("foreign key (org_id) references organizations(id) ON UPDATE SET NULL ON DELETE SET NULL");
createGlobalReferenceSetsTable.append(")");
String globalReferenceSetsIdx1 = "CREATE INDEX IF NOT EXISTS global_reference_sets_org_id ON global_reference_sets (org_id)";
@@ -293,8 +293,8 @@ public final class SqliteEamDbSettings {
createGlobalFilesTable.append("value text NOT NULL,");
createGlobalFilesTable.append("known_status text NOT NULL,");
createGlobalFilesTable.append("comment text NOT NULL,");
createGlobalFilesTable.append("CONSTRAINT global_files_multi_unique UNIQUE(global_reference_set_id, value)");
createGlobalFilesTable.append("foreign key (global_reference_set_id) references global_reference_sets(id) on update set null on delete set null");
createGlobalFilesTable.append("CONSTRAINT global_files_multi_unique UNIQUE(global_reference_set_id, value) ON CONFLICT IGNORE,");
createGlobalFilesTable.append("foreign key (global_reference_set_id) references global_reference_sets(id) ON UPDATE SET NULL ON DELETE SET NULL");
createGlobalFilesTable.append(")");
String globalFilesIdx1 = "CREATE INDEX IF NOT EXISTS global_files_value ON global_files (value)";
@@ -321,9 +321,9 @@ public final class SqliteEamDbSettings {
createArtifactInstancesTableTemplate.append("file_path text NOT NULL,");
createArtifactInstancesTableTemplate.append("known_status text NOT NULL,");
createArtifactInstancesTableTemplate.append("comment text NOT NULL,");
createArtifactInstancesTableTemplate.append("CONSTRAINT %s_instances_multi_unique UNIQUE(case_id, data_source_id, value, file_path),");
createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) on update set null on delete set null,");
createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) on update set null on delete set null");
createArtifactInstancesTableTemplate.append("CONSTRAINT %s_instances_multi_unique UNIQUE(case_id, data_source_id, value, file_path) ON CONFLICT IGNORE,");
createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL");
createArtifactInstancesTableTemplate.append(")");
// TODO: do we need any more indices?