working through classpath issues

IntegrationTesting/build.xml

@@ -2,11 +2,14 @@
 <!-- You may freely edit this file. See harness/README in the NetBeans platform -->
 <!-- for some information on what you could do (e.g. targets to override). -->
 <!-- If you delete this file and reopen the project it will be recreated. -->
-<project name="org.sleuthkit.autopsy.integrationtesting" default="netbeans" basedir=".">
+<project name="org.sleuthkit.autopsy.integrationtesting" default="netbeans" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant" >
     <description>Builds, tests, and runs the project org.sleuthkit.autopsy.integrationtesting.</description>
     <import file="nbproject/build-impl.xml"/>
+    <import file="../BootstrapIvy.xml"/>
     <property name="regression" value="qa-functional"/>
- 
+    <property name="modules.dir" value="${basedir}/release/modules/" />
+    <property name="ext.dir" value="${modules.dir}/ext" />
+
     <target name="check-args"> 
 <!--        <fail message="Missing required argument: img_path" unless="img_path"/>
         <fail message="Missing required argument: gold_path" unless="gold_path"/>
@@ -34,9 +37,24 @@
         </path>
     </target>
 
-    <target name="regression-test" depends="check-args,init,test-init,test-build, manifest-classpath" if="exists.test.qa-functional.src.dir">
+    <target name="get-deps" depends="init-ivy">
+        <mkdir dir="${ext.dir}"/>
+        <ivy:resolve log="quiet"/>
+        <ivy:retrieve conf="integrationtesting" pattern="${ext.dir}/[artifact]-[revision](-[classifier]).[ext]" />
+    </target>
+
+    <target name="init" depends="get-deps,harness.init"/>
+
+    <target name="integration-test" depends="check-args,init,test-init,test-build,manifest-classpath" if="exists.test.qa-functional.src.dir">
         <test test.type="${regression}"/>
-        <delete file="allJarsInUse.jar"/>
+        <!--<delete file="allJarsInUse.jar"/>-->
+    </target>
+
+    <target name="clean" depends="projectized-common.clean">
+        <!--Override clean to delete jars, etc downloaded with Ivy, 
+        or copied in from thirdparty folder.  This way we don't end up with 
+        out-of-date/unneeded stuff in the installer-->
+        <delete dir="${basedir}/release/"/>
     </target>
     
     <macrodef name="test">
@@ -45,7 +63,7 @@
 
         <sequential>
             <property name="test.config" value="default"/>
-            <property name="test.config.default.includes" value="**/*Test.class"/>
+            <property name="test.config.default.includes" value="**/org/sleuthkit/autopsy/integrationtesting/MainTestRunner.class"/>
             <property name="test.config.${test.config}.includes" value="NOTHING"/>
             <metaproperty name="test.includes" value="test.config.${test.config}.includes"/>
             <property name="test.config.${test.config}.excludes" value=""/>
@@ -56,13 +74,14 @@
                     <fileset dir="${build.test.@{test.type}.classes.dir}" includes="${test.includes}" excludes="${test.excludes}"/>
                 </batchtest>
                 <classpath refid="test.classpath"/>
+                <!--<classpath refid="test.@{test.type}.run.cp"/>-->
                 <syspropertyset refid="test.@{test.type}.properties"/>
                 <jvmarg line="${test.bootclasspath.prepend.args}"/>
                 <jvmarg line="${test.run.args}"/>
                 <!--  should be in sync with project.properties and build.xml of autopsy main project -->
                 <!-- disable for now, causes issues with Java 7 -->
                 <!-- <jvmarg line="-J-Xms24m -J-Xmx512m -J-XX:MaxPermSize=128M -J-Xverify:none"/>  -->
-                <sysproperty key="img_path" value="${img_path}"/>
+<!--                <sysproperty key="img_path" value="${img_path}"/>
                 <sysproperty key="gold_path" value="${gold_path}"/>
                 <sysproperty key="out_path" value="${out_path}"/>
                 <sysproperty key="known_bad_path" value="${known_bad_path}"/>
@@ -76,7 +95,7 @@
                 <sysproperty key="solrPort" value="${solrPort}"/>
                 <sysproperty key="messageServiceHost" value="${messageServiceHost}"/>
                 <sysproperty key="messageServicePort" value="${messageServicePort}"/>
-                <sysproperty key="isMultiUser" value="${isMultiUser}"/>
+                <sysproperty key="isMultiUser" value="${isMultiUser}"/>-->
                 <!--needed to have tests NOT to steal focus when running, works in latest apple jdk update only.-->
                 <sysproperty key="apple.awt.UIElement" value="@{disable.apple.ui}"/>
                 <formatter type="brief" usefile="false"/>

IntegrationTesting/ivy.xml

@@ -0,0 +1,15 @@
+<ivy-module version="2.0">
+    <info organisation="org.sleuthkit.autopsy" module="integrationtesting"/>
+    <configurations >
+        <!-- module dependencies -->
+        <conf name="integrationtesting"/>
+
+    </configurations>
+    <dependencies >
+        <!-- for yaml reading/writing -->
+        <dependency org="org.yaml" name="snakeyaml" rev="1.27"/>
+
+        <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
+        <dependency org="com.google.code.gson" name="gson" rev="2.8.6"/>
+    </dependencies>
+</ivy-module>

IntegrationTesting/ivysettings.xml

@@ -0,0 +1,9 @@
+<ivysettings> 
+  <settings defaultResolver="main"/>
+  <resolvers>
+    <chain name="main">
+        <ibiblio name="central" root="https://repo1.maven.org/maven2" m2compatible="true"/>
+        <ibiblio name="maven.restlet.org" root="http://maven.restlet.com" m2compatible="true" />  
+    </chain>
+  </resolvers>
+</ivysettings>

IntegrationTesting/nbproject/platform.properties

@@ -0,0 +1,16 @@
+cluster.path=\
+    ${nbplatform.active.dir}/platform
+disabled.modules=\
+    org.netbeans.api.visual,\
+    org.netbeans.core.execution,\
+    org.netbeans.core.multiview,\
+    org.netbeans.libs.jsr223,\
+    org.netbeans.modules.autoupdate.services,\
+    org.netbeans.modules.autoupdate.ui,\
+    org.netbeans.modules.core.kit,\
+    org.netbeans.modules.favorites,\
+    org.openide.compat,\
+    org.openide.execution,\
+    org.openide.options,\
+    org.openide.util.enumerations
+nbplatform.active=default

IntegrationTesting/nbproject/project.properties

@@ -1,3 +1,21 @@
+file.reference.commons-collections-3.2.1.jar=release\\modules\\ext\\commons-collections-3.2.1.jar
+file.reference.commons-io-2.5.jar=release\\modules\\ext\\commons-io-2.5.jar
+file.reference.commons-lang-2.4.jar=release\\modules\\ext\\commons-lang-2.4.jar
+file.reference.commons-lang3-3.4.jar=release\\modules\\ext\\commons-lang3-3.4.jar
+file.reference.hamcrest-core-1.3.jar=release\\modules\\ext\\hamcrest-core-1.3.jar
+file.reference.jackson-annotations-2.11.3.jar=release\\modules\\ext\\jackson-annotations-2.11.3.jar
+file.reference.jackson-annotations-2.9.7.jar=release\\modules\\ext\\jackson-annotations-2.9.7.jar
+file.reference.jackson-core-2.11.3.jar=release\\modules\\ext\\jackson-core-2.11.3.jar
+file.reference.jackson-core-2.9.7.jar=release\\modules\\ext\\jackson-core-2.9.7.jar
+file.reference.jackson-databind-2.11.3.jar=release\\modules\\ext\\jackson-databind-2.11.3.jar
+file.reference.jackson-databind-2.9.7.jar=release\\modules\\ext\\jackson-databind-2.9.7.jar
+file.reference.jackson-dataformat-yaml-2.11.3.jar=release\\modules\\ext\\jackson-dataformat-yaml-2.11.3.jar
+file.reference.jackson-dataformat-yaml-2.9.7.jar=release\\modules\\ext\\jackson-dataformat-yaml-2.9.7.jar
+file.reference.joda-time-2.10.1.jar=release\\modules\\ext\\joda-time-2.10.1.jar
+file.reference.junit-4.12.jar=release\\modules\\ext\\junit-4.12.jar
+file.reference.oro-2.0.8.jar=release\\modules\\ext\\oro-2.0.8.jar
+file.reference.snakeyaml-1.27.jar=release\\modules\\ext\\snakeyaml-1.27.jar
+file.reference.velocity-1.6.2.jar=release\\modules\\ext\\velocity-1.6.2.jar
 javac.source=1.8
 javac.compilerargs=-Xlint -Xlint:-serial
 license.file=../LICENSE-2.0.txt

IntegrationTesting/nbproject/project.xml

@@ -101,6 +101,46 @@
             <public-packages>
                 <package>org.sleuthkit.autopsy.integrationtesting</package>
             </public-packages>
+            <class-path-extension>
+                <runtime-relative-path>ext/hamcrest-core-1.3.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\hamcrest-core-1.3.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/velocity-1.6.2.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\velocity-1.6.2.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/commons-lang3-3.4.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\commons-lang3-3.4.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/commons-lang-2.4.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\commons-lang-2.4.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/commons-collections-3.2.1.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\commons-collections-3.2.1.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/oro-2.0.8.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\oro-2.0.8.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/joda-time-2.10.1.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\joda-time-2.10.1.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/junit-4.12.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\junit-4.12.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/commons-io-2.5.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\commons-io-2.5.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/snakeyaml-1.27.jar</runtime-relative-path>
+                <binary-origin>release\modules\ext\snakeyaml-1.27.jar</binary-origin>
+            </class-path-extension>
         </data>
     </configuration>
 </project>

IntegrationTesting/test/qa-functional/src/org/sleuthkit/autopsy/integrationtesting/MainTestRunner.java

@@ -18,10 +18,12 @@
  */
 package org.sleuthkit.autopsy.integrationtesting;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import org.sleuthkit.autopsy.integrationtesting.config.IntegrationTestConfig;
@@ -58,6 +60,7 @@ import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer;
 import org.sleuthkit.autopsy.testutils.CaseUtils;
 import org.sleuthkit.autopsy.testutils.IngestUtils;
 import org.sleuthkit.datamodel.TskCoreException;
+import org.yaml.snakeyaml.Yaml;
 
 /**
  * Main entry point for running integration tests. Handles processing
@@ -67,7 +70,7 @@ import org.sleuthkit.datamodel.TskCoreException;
 public class MainTestRunner extends TestCase {
 
     private static final Logger logger = Logger.getLogger(MainTestRunner.class.getName()); // DO NOT USE AUTOPSY LOGGER
-    private static final String CONFIG_FILE_KEY = "CONFIG_FILE_KEY";
+    private static final String CONFIG_FILE_KEY = "configFileKey";
     private static final IngestModuleFactoryService ingestFactoryService = new IngestModuleFactoryService();
     private static final IngestType DEFAULT_INGEST_TYPE = IngestType.ALL_MODULES;
 
@@ -213,10 +216,11 @@ public class MainTestRunner extends TestCase {
     }
 
     private IntegrationTestConfig getConfigFromFile(String filePath) throws IOException {
-        ObjectMapper om = new ObjectMapper();
-        try (FileInputStream jsonSrc = new FileInputStream(filePath)) {
-            return om.readValue(jsonSrc, IntegrationTestConfig.class);
-        }
+        //return new IntegrationTestConfig("C:\\Users\\gregd\\Desktop\\testoutput", "C:\\Users\\gregd\\Desktop\\testoutput", Collections.emptyList());
+        GsonBuilder builder = new GsonBuilder();
+        builder.registerTypeAdapter(IntegrationTestConfig.class, IntegrationTestConfig.DESERIALIZER);
+        Gson gson = builder.create();
+        return gson.fromJson(new FileReader(new File(filePath)), IntegrationTestConfig.class);
     }
 
     private void runIntegrationTests(IntegrationTestConfig config, CaseConfig caseConfig, CaseType caseType) {
@@ -301,10 +305,12 @@ public class MainTestRunner extends TestCase {
     private void serializeFile(OutputResults results, String outputFolder, String caseName, String caseType) {
         String outputExtension = ".yml";
         Path outputPath = Paths.get(outputFolder, String.format("%s-%s%s", caseName, caseType, outputExtension));
-        ObjectMapper om = new ObjectMapper(new YAMLFactory());
+        Yaml yaml = new Yaml();
 
         try {
-            om.writeValue(outputPath.toFile(), results.getSerializableData());
+
+            FileWriter writer = new FileWriter(outputPath.toFile());
+            yaml.dump(results.getSerializableData(), writer);
         } catch (IOException ex) {
             logger.log(Level.WARNING, "There was an error writing results to outputPath: " + outputPath, ex);
         }

IntegrationTesting/test/qa-functional/src/org/sleuthkit/autopsy/integrationtesting/config/IntegrationTestConfig.java

@@ -18,6 +18,13 @@
  */
 package org.sleuthkit.autopsy.integrationtesting.config;
 
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.reflect.TypeToken;
+import java.lang.reflect.Type;
 import java.util.List;
 
 /**
@@ -25,6 +32,21 @@ import java.util.List;
  * parameters, datasource locations, cases to create, tests to run, etc.
  */
 public class IntegrationTestConfig {
+    private static final Type listOfCasesType = new TypeToken<List<CaseConfig>>(){}.getType();
+        
+    public static final JsonDeserializer<IntegrationTestConfig> DESERIALIZER = new JsonDeserializer<IntegrationTestConfig>() {
+        @Override
+        public IntegrationTestConfig deserialize(JsonElement je, Type type, JsonDeserializationContext jdc) throws JsonParseException {
+            JsonObject jObj = je.getAsJsonObject();
+            String rootCaseOutputPath = jObj.get("rootTestOutputPath").getAsString();
+            String rootTestOutputPath = jObj.get("rootTestOutputPath").getAsString();
+            List<CaseConfig> cases = jdc.deserialize(jObj.get("cases"), listOfCasesType);
+            
+            return new IntegrationTestConfig(rootCaseOutputPath, rootTestOutputPath, cases);
+        }
+    };
+    
+    
     private final String rootCaseOutputPath;
     private final String rootTestOutputPath;
     private final List<CaseConfig> cases;

IntegrationTesting/test/qa-functional/src/org/sleuthkit/autopsy/integrationtesting/config/TestingConfig.java

@@ -18,8 +18,6 @@
  */
 package org.sleuthkit.autopsy.integrationtesting.config;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
@@ -43,8 +41,7 @@ public class TestingConfig {
                 .collect(Collectors.toSet());
     }
     
-    @JsonCreator
-    public TestingConfig(@JsonProperty() List<String> excludeAllExcept, @JsonProperty() List<String> includeAllExcept) {
+    public TestingConfig(List<String> excludeAllExcept, List<String> includeAllExcept) {
         this.excludeAllExcept = convert(excludeAllExcept);
         this.includeAllExcept = convert(includeAllExcept);
     }