From 2e36c8a4d6afe44745f391bdc7083d0458daa2e1 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 12 Feb 2020 10:06:02 -0500 Subject: [PATCH] Getting ClousSolrClient to work --- .../CoordinationService.java | 4 +- .../autopsy/core/UserPreferences.java | 18 +++++ .../keywordsearch/Bundle.properties-MERGED | 1 + .../autopsy/keywordsearch/Server.java | 80 ++++++++++++++----- .../keywordsearch/SolrSearchService.java | 31 +++++-- 5 files changed, 105 insertions(+), 29 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coordinationservice/CoordinationService.java b/Core/src/org/sleuthkit/autopsy/coordinationservice/CoordinationService.java index 9bd5710980..9788a94468 100644 --- a/Core/src/org/sleuthkit/autopsy/coordinationservice/CoordinationService.java +++ b/Core/src/org/sleuthkit/autopsy/coordinationservice/CoordinationService.java @@ -74,8 +74,8 @@ public final class CoordinationService { private static boolean isZooKeeperAccessible() throws InterruptedException, IOException { boolean result = false; Object workerThreadWaitNotifyLock = new Object(); - int zooKeeperServerPort = Integer.valueOf(UserPreferences.getIndexingServerPort()) + PORT_OFFSET; - String connectString = UserPreferences.getIndexingServerHost() + ":" + zooKeeperServerPort; + // ELTODO int zooKeeperServerPort = Integer.valueOf(UserPreferences.getIndexingServerPort()) + PORT_OFFSET; + String connectString = UserPreferences.getZkServerHost() + ":" + UserPreferences.getZkServerPort(); ZooKeeper zooKeeper = new ZooKeeper(connectString, ZOOKEEPER_SESSION_TIMEOUT_MILLIS, (WatchedEvent event) -> { synchronized (workerThreadWaitNotifyLock) { diff --git a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java index 409e27f080..426ff45db9 100644 --- a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java +++ b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java @@ -56,6 +56,8 @@ public final class UserPreferences { public static final String EXTERNAL_DATABASE_TYPE = "ExternalDatabaseType"; //NON-NLS public static final String INDEXING_SERVER_HOST = "IndexingServerHost"; //NON-NLS public static final String INDEXING_SERVER_PORT = "IndexingServerPort"; //NON-NLS + public static final String ZK_SERVER_HOST = "ZookeeperServerHost"; //NON-NLS + public static final String ZK_SERVER_PORT = "ZookeeperServerPort"; //NON-NLS private static final String MESSAGE_SERVICE_PASSWORD = "MessageServicePassword"; //NON-NLS private static final String MESSAGE_SERVICE_USER = "MessageServiceUser"; //NON-NLS private static final String MESSAGE_SERVICE_HOST = "MessageServiceHost"; //NON-NLS @@ -337,6 +339,22 @@ public final class UserPreferences { public static void setIndexingServerPort(int port) { preferences.putInt(INDEXING_SERVER_PORT, port); } + + public static String getZkServerHost() { + return preferences.get(ZK_SERVER_HOST, ""); + } + + public static void setZkServerHost(String hostName) { + preferences.put(ZK_SERVER_HOST, hostName); + } + + public static String getZkServerPort() { + return preferences.get(ZK_SERVER_PORT, "9983"); + } + + public static void setZkServerPort(int port) { + preferences.putInt(ZK_SERVER_PORT, port); + } public static void setTextTranslatorName(String textTranslatorName) { preferences.put(TEXT_TRANSLATOR_NAME, textTranslatorName); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index d2fe0dd2db..a454b63586 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -222,6 +222,7 @@ NoOpenCoreException.err.noOpenSorlCore.msg=No currently open Solr core. SearchRunner.query.exception.msg=Error performing query: # {0} - core name Server.deleteCore.exception.msg=Failed to delete Solr core {0} +Server.exceptionMessage.unableToCreateCollection=Unable to create Solr collection Server.start.exception.cantStartSolr.msg=Could not start Solr server process Server.start.exception.cantStartSolr.msg2=Could not start Solr server process Server.isRunning.exception.errCheckSolrRunning.msg=Error checking if Solr server is running diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 039a711f3b..44744461ba 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -50,13 +50,15 @@ import org.apache.commons.io.FileUtils; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; // for Single User mode +import org.apache.solr.client.solrj.impl.CloudSolrClient; // for Solr Cloud mode import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.response.CoreAdminResponse; import java.util.Properties; +import java.util.concurrent.TimeoutException; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.TermsResponse; @@ -239,7 +241,8 @@ public class Server { // A reference to the Solr server we are currently connected to for the Case. // This could be a local or remote server. - private HttpSolrClient currentSolrServer; +// ELTODO private HttpSolrClient currentSolrServer; + private CloudSolrClient currentSolrServer; private Collection currentCollection; private final ReentrantReadWriteLock currentCoreLock; @@ -791,13 +794,19 @@ public class Server { "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",}) void deleteCore(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException { try { - HttpSolrClient solrServer; + IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory()); + List solrUrls = new ArrayList<>(); + solrUrls.add("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); + CloudSolrClient solrServer = new CloudSolrClient.Builder(solrUrls).build(); + + //HttpSolrClient solrServer; if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) { Integer localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT)); - solrServer = new HttpSolrClient.Builder("http://localhost:" + localSolrServerPort + "/solr").build(); //NON-NLS + // ELTODO solrServer = new HttpSolrClient.Builder("http://localhost:" + localSolrServerPort + "/solr").build(); //NON-NLS } else { - IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory()); - solrServer = new HttpSolrClient.Builder("http://" + properties.getHost() + ":" + properties.getPort() + "/solr").build(); //NON-NLS + //IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory()); + // ELTODO solrServer = new CloudSolrClient.Builder().withZkHost(zkHosts).build(); + // ELTODO solrServer = new HttpSolrClient.Builder("http://" + properties.getHost() + ":" + properties.getPort() + "/solr").build(); //NON-NLS } connectToSolrServer(solrServer); @@ -843,7 +852,8 @@ public class Server { try { if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) { - currentSolrServer = this.localSolrServer; + // ELTODO make embedded Solr work with localSolrServer + // currentSolrServer = this.localSolrServer; // check if the embedded Solr server is running if (!this.isEmbeddedSolrRunning()) { @@ -851,13 +861,30 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg")); } - connectToEbmeddedSolrServer(currentSolrServer); + // ELTODO connectToEbmeddedSolrServer(currentSolrServer); } else { IndexingServerProperties properties = getMultiUserServerProperties(theCase.getCaseDirectory()); - currentSolrServer = new HttpSolrClient.Builder("http://" + properties.getHost() + ":" + properties.getPort() + "/solr").build(); //NON-NLS + List solrUrls = new ArrayList<>(); + solrUrls.add("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); + solrUrls.add("http://review1:" + properties.getPort() + "/solr"); + solrUrls.add("http://ingest9:" + properties.getPort() + "/solr"); + currentSolrServer = new CloudSolrClient.Builder(solrUrls).build(); // (new CloudSolrClient.Builder(solrUrls)); + currentSolrServer.connect(10, TimeUnit.SECONDS); + //currentSolrServer.setZkClientTimeout(30000); + //currentSolrServer.setZkConnectTimeout(30000); + + /* ELTODO + String zkHost = "localhost"; //properties.getHost(); + String zkPort = "9983"; + List zkHosts = new ArrayList<>(); + zkHosts.add(zkHost + ":" + zkPort); + currentSolrServer = new CloudSolrClient.Builder(zkHosts, Optional.empty()).build(); + currentSolrServer.connect();*/ + + //currentSolrServer = new HttpSolrClient.Builder("http://" + properties.getHost() + ":" + properties.getPort() + "/solr").build(); //NON-NLS connectToSolrServer(currentSolrServer); } - } catch (SolrServerException | IOException ex) { + } catch (/* ELTODO SolrServerException | IOException*/ Exception ex) { throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg", ex.getLocalizedMessage()), ex); } @@ -874,7 +901,7 @@ public class Server { * The core either does not exist or it is not loaded. Make a * request that will cause the core to be created. */ - int NUM_RETRIES = 3; + int NUM_RETRIES = 10; boolean doRetry = false; for (int reTryAttempt = 0; reTryAttempt < NUM_RETRIES; reTryAttempt++) { try { @@ -923,7 +950,7 @@ public class Server { } } - return new Collection(collectionName, theCase.getCaseType(), index); + return new Collection(collectionName, theCase, index); } catch (Exception ex) { logger.log(Level.SEVERE, "Exception during Solr collection creation.", ex); //NON-NLS @@ -1109,7 +1136,7 @@ public class Server { } } } - + /** * Helper class to store the current server properties */ @@ -1527,7 +1554,8 @@ public class Server { * @throws SolrServerException * @throws IOException */ - void connectToSolrServer(HttpSolrClient solrServer) throws SolrServerException, IOException { + // ELTODO void connectToSolrServer(HttpSolrClient solrServer) throws SolrServerException, IOException { + void connectToSolrServer(CloudSolrClient solrServer) throws SolrServerException, IOException { TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check"); CollectionAdminRequest.ClusterStatus statusRequest = CollectionAdminRequest.getClusterStatus(); CollectionAdminResponse statusResponse = statusRequest.process(solrServer); @@ -1655,23 +1683,37 @@ public class Server { // the server to access a collection needs to be built from a URL with the // collection in it, and is only good for collection-specific operations - private final HttpSolrClient solrClient; + private final CloudSolrClient solrClient; private final int QUERY_TIMEOUT_MILLISECONDS = 86400000; // 24 Hours = 86,400,000 Milliseconds - private Collection(String name, CaseType caseType, Index index) { + private Collection(String name, Case theCase, Index index) throws TimeoutException, InterruptedException { this.name = name; - this.caseType = caseType; + this.caseType = theCase.getCaseType(); this.textIndex = index; - this.solrClient = new HttpSolrClient.Builder(currentSolrServer.getBaseURL() + "/" + name) + // ELTODO get this from some configuration or UI + //IndexingServerProperties properties = getMultiUserServerProperties(Case.getCurrentCase().getCaseDirectory()); + + IndexingServerProperties properties = getMultiUserServerProperties(theCase.getCaseDirectory()); + List solrUrls = new ArrayList<>(); + solrUrls.add("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); + solrUrls.add("http://review1:" + properties.getPort() + "/solr"); + solrUrls.add("http://ingest9:" + properties.getPort() + "/solr"); + solrClient = new CloudSolrClient.Builder(solrUrls).build(); + solrClient.setDefaultCollection(name); + solrClient.connect(10, TimeUnit.SECONDS); + // ELTODO solrClient.withSocketTimeout(QUERY_TIMEOUT_MILLISECONDS); + // ELTODO solrClient.allowCompression(true); + /* ELTODO this.solrClient = new HttpSolrClient.Builder(currentSolrServer.getBaseURL() + "/" + name) .withSocketTimeout(QUERY_TIMEOUT_MILLISECONDS) .allowCompression(true) // allowCompression defaults to false. Server side must support gzip or deflate for this to have any effect. .build(); //NON-NLS + */ //TODO test these settings //solrCore.setConnectionTimeout(1000); - solrClient.setFollowRedirects(false); // defaults to false + // ELTODO solrClient.setFollowRedirects(false); // defaults to false solrClient.setParser(new XMLResponseParser()); // binary parser is used by default diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 73841d40f0..d8a9dbe813 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -27,6 +27,7 @@ import java.net.InetAddress; import java.util.ArrayList; import java.util.List; import java.util.MissingResourceException; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import javax.swing.JDialog; import javax.swing.JOptionPane; @@ -34,6 +35,7 @@ import javax.swing.SwingUtilities; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.math.NumberUtils; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; @@ -45,6 +47,7 @@ import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import static org.sleuthkit.autopsy.keywordsearch.Server.getMultiUserServerProperties; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; import org.sleuthkit.autopsy.progress.ProgressIndicator; @@ -154,16 +157,28 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { */ @Override public void tryConnect(String host, int port) throws KeywordSearchServiceException { - HttpSolrClient solrServer = null; + // ELTODO HttpSolrClient solrServer = null; + CloudSolrClient solrServer = null; if (host == null || host.isEmpty()) { throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.MissingHostname")); //NON-NLS } try { - solrServer = new HttpSolrClient.Builder("http://" + host + ":" + Integer.toString(port) + "/solr").build(); //NON-NLS - KeywordSearch.getServer().connectToSolrServer(solrServer); - } catch (SolrServerException ex) { - throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort")); //NON-NLS - } catch (IOException ex) { + // ELTODO solrServer = new HttpSolrClient.Builder("http://" + host + ":" + Integer.toString(port) + "/solr").build(); //NON-NLS + List solrUrls = new ArrayList<>(); + solrUrls.add("http://" + host + ":" + port + "/solr"); + solrUrls.add("http://review1:" + port + "/solr"); + solrUrls.add("http://ingest9:" + port + "/solr"); + solrServer = new CloudSolrClient.Builder(solrUrls).build(); + //solrServer = new CloudSolrClient.Builder().withZkHost(zkHosts).build(); + //solrServer.setZkClientTimeout(30000); + //solrServer.setZkConnectTimeout(30000); + solrServer.connect(10, TimeUnit.SECONDS); + + //solrServer = new CloudSolrClient(zkHosts); + // ELTODO KeywordSearch.getServer().connectToSolrServer(solrServer); + /* ELTODO } catch (SolrServerException ex) { + throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort")); //NON-NLS*/ + } catch (/*ELTODO IOException*/ Exception ex) { String result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS String message = ex.getCause().getMessage().toLowerCase(); if (message.startsWith(SERVER_REFUSED_CONNECTION)) { @@ -182,10 +197,10 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.Hostname"); //NON-NLS } throw new KeywordSearchServiceException(result); - } catch (NumberFormatException ex) { + /* ELTODO} catch (NumberFormatException ex) { throw new KeywordSearchServiceException(Bundle.SolrConnectionCheck_Port()); } catch (IllegalArgumentException ex) { - throw new KeywordSearchServiceException(ex.getMessage()); + throw new KeywordSearchServiceException(ex.getMessage());*/ } finally { if (null != solrServer) { try {