From 994ca22c8bbb56b01655dedcf3a0ddda3ab2b590 Mon Sep 17 00:00:00 2001 From: jmillman Date: Wed, 27 Jul 2016 15:21:08 -0400 Subject: [PATCH] modified to limit indexed text viewer to relavent chunk_id --- .../sleuthkit/autopsy/datamodel/Accounts.java | 13 +- .../autopsy/keywordsearch/AccountsText.java | 378 ++++++++++++++++++ .../keywordsearch/ExtractedContentViewer.java | 54 +-- .../keywordsearch/HighlightedText.java | 126 ++---- 4 files changed, 442 insertions(+), 129 deletions(-) create mode 100644 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AccountsText.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java index 7d206d8946..d45b196331 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java @@ -43,6 +43,7 @@ import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -460,7 +461,13 @@ public class Accounts extends Observable implements AutopsyVisitableItem { @Override protected Node createNodeForKey(FileWithCCN key) { try { - return new FileWithCCNNode(key, skCase.getAbstractFileById(key.getObjID())); + List artifacts = new ArrayList<>(); + for (long artId : key.artifactIDS) { + artifacts.add(skCase.getBlackboardArtifact(artId)); + } + AbstractFile abstractFileById = skCase.getAbstractFileById(key.getObjID()); + artifacts.add(abstractFileById); + return new FileWithCCNNode(key, abstractFileById, artifacts.toArray()); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error getting content for file with ccn hits.", ex); //NON-NLS return null; @@ -485,8 +492,8 @@ public class Accounts extends Observable implements AutopsyVisitableItem { "# {0} - raw file name", "# {1} - solr chunk id", "Accounts.FileWithCCNNode.unallocatedSpaceFile.displayName={0}_chunk_{1}"}) - private FileWithCCNNode(FileWithCCN key, Content content) { - super(Children.LEAF, Lookups.singleton(content)); + private FileWithCCNNode(FileWithCCN key, Content content, Object[] lookupContents) { + super(Children.LEAF, Lookups.fixed(lookupContents)); this.fileKey = key; this.fileName = (key.getSolrDocmentID() == null) ? content.getName() diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AccountsText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AccountsText.java new file mode 100644 index 0000000000..37ca698454 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AccountsText.java @@ -0,0 +1,378 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2016 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.datamodel.TextMarkupLookup; + +/** + * Highlights account hits for a given document. Knows about pages and such for + * the content viewer. + */ +class AccountsText implements IndexedText, TextMarkupLookup { + + private static final Logger LOGGER = Logger.getLogger(AccountsText.class.getName()); + private static final String HIGHLIGHT_PRE = ""; //NON-NLS + private static final String HIGHLIGHT_POST = ""; //NON-NLS + private static final String ANCHOR_PREFIX = AccountsText.class.getName() + "_"; + + private final String solrDocumentId; + private final Set keywords = new HashSet<>(); + private final Server solrServer; + private int numberPagesForFile = 0; + private int currentPage = 0; + private boolean hasChunks = false; + //stores all pages/chunks that have hits as key, and number of hits as a value, or 0 if yet unknown + private final LinkedHashMap numberOfHitsPerPage = new LinkedHashMap<>(); + //stored page num -> current hit number mapping + private final HashMap currentHitPerPage = new HashMap<>(); + private final List pages = new ArrayList<>(); + private boolean isPageInfoLoaded = false; + private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT); + private String displayName; + private final long solrObjectId; + private final Integer chunkId; + + synchronized String getDisplayName() { + if (StringUtils.isBlank(displayName)) { + return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.toString"); + } else { + return displayName; + } + } + + synchronized void setDisplayName(String displayName) { + this.displayName = displayName; + } + + AccountsText(String objectId, Set keywords) { + this.solrDocumentId = objectId; + this.keywords.addAll(keywords); + this.solrServer = KeywordSearch.getServer(); + + final int separatorIndex = solrDocumentId.indexOf(Server.ID_CHUNK_SEP); + if (-1 != separatorIndex) { + this.solrObjectId = Long.parseLong(solrDocumentId.substring(0, separatorIndex)); + this.chunkId = Integer.parseInt(solrDocumentId.substring(separatorIndex + 1)); + } else { + this.solrObjectId = Long.parseLong(solrDocumentId); + this.chunkId = null; + } + } + + long getObjectId() { + return this.solrObjectId; + } + + @Override + public int getNumberPages() { + return this.numberPagesForFile; + } + + @Override + public int getCurrentPage() { + return this.currentPage; + } + + @Override + public boolean hasNextPage() { + return pages.indexOf(this.currentPage) < pages.size() - 1; + + } + + @Override + public boolean hasPreviousPage() { + return pages.indexOf(this.currentPage) > 0; + + } + + @Override + public int nextPage() { + if (hasNextPage()) { + currentPage = pages.get(pages.indexOf(this.currentPage) + 1); + return currentPage; + } else { + throw new IllegalStateException(NbBundle.getMessage(AccountsText.class, "HighlightedMatchesSource.nextPage.exception.msg")); + } + } + + @Override + public int previousPage() { + if (hasPreviousPage()) { + currentPage = pages.get(pages.indexOf(this.currentPage) - 1); + return currentPage; + } else { + throw new IllegalStateException(NbBundle.getMessage(AccountsText.class, "HighlightedMatchesSource.previousPage.exception.msg")); + } + } + + @Override + public boolean hasNextItem() { + if (this.currentHitPerPage.containsKey(currentPage)) { + return this.currentHitPerPage.get(currentPage) < this.numberOfHitsPerPage.get(currentPage); + } else { + return false; + } + } + + @Override + public boolean hasPreviousItem() { + if (this.currentHitPerPage.containsKey(currentPage)) { + return this.currentHitPerPage.get(currentPage) > 1; + } else { + return false; + } + } + + @Override + public int nextItem() { + if (hasNextItem()) { + return currentHitPerPage.merge(currentPage, 1, Integer::sum); + } else { + throw new IllegalStateException(NbBundle.getMessage(AccountsText.class, "HighlightedMatchesSource.nextItem.exception.msg")); + } + } + + @Override + public int previousItem() { + if (hasPreviousItem()) { + return currentHitPerPage.merge(currentPage, -1, Integer::sum); + } else { + throw new IllegalStateException(NbBundle.getMessage(AccountsText.class, "HighlightedMatchesSource.previousItem.exception.msg")); + } + } + + @Override + public int currentItem() { + if (this.currentHitPerPage.containsKey(currentPage)) { + return currentHitPerPage.get(currentPage); + } else { + return 0; + } + } + + @Override + public LinkedHashMap getHitsPages() { + return this.numberOfHitsPerPage; + } + + /** + * The main goal of this method is to figure out which pages / chunks have + * hits. + */ + synchronized private void loadPageInfo() { + if (isPageInfoLoaded) { + return; + } + if (chunkId != null) { + this.numberPagesForFile = 1; + } else { + try { + this.numberPagesForFile = solrServer.queryNumFileChunks(this.solrObjectId); + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + LOGGER.log(Level.WARNING, "Could not get number pages for content: " + this.solrDocumentId); //NON-NLS + return; + } + } + + if (this.numberPagesForFile <= 1) { + hasChunks = false; + //no chunks + this.numberPagesForFile = 1; + this.currentPage = chunkId; + numberOfHitsPerPage.put(chunkId, 0); + pages.add(chunkId); + currentHitPerPage.put(chunkId, 0); + } else { + hasChunks = true; + //if has chunks, get pages with hits + TreeSet sortedPagesWithHits = new TreeSet<>(); + //extract pages of interest, sorted + + SolrQuery q = new SolrQuery(); + q.setShowDebugInfo(DEBUG); //debug + String query = keywords.stream().map(keyword -> "/.*" + KeywordSearchUtil.escapeLuceneQuery(keyword) + ".*/").collect(Collectors.joining(" ")); + q.setQuery(LuceneQuery.HIGHLIGHT_FIELD_REGEX + ":" + query); + q.setFields("id"); + if (chunkId == null) { + q.addFilterQuery(Server.Schema.ID.toString() + ":" + this.solrObjectId + "_*"); + } else { + q.addFilterQuery(Server.Schema.ID.toString() + ":" + this.solrDocumentId); + } + try { + QueryResponse response = solrServer.query(q, METHOD.POST); + for (SolrDocument resultDoc : response.getResults()) { + final String resultDocumentId = resultDoc.getFieldValue(Server.Schema.ID.toString()).toString(); + // Put the solr chunk id in the map + final int separatorIndex = resultDocumentId.indexOf(Server.ID_CHUNK_SEP); + if (-1 != separatorIndex) { + sortedPagesWithHits.add(Integer.parseInt(resultDocumentId.substring(separatorIndex + 1))); + } else { + sortedPagesWithHits.add(0); + } + } + + } catch (KeywordSearchModuleException | NoOpenCoreException | NumberFormatException ex) { + LOGGER.log(Level.WARNING, "Error executing Solr highlighting query: " + keywords, ex); //NON-NLS + } + + //set page to first page having highlights + if (sortedPagesWithHits.isEmpty()) { + this.currentPage = 0; + } else { + this.currentPage = sortedPagesWithHits.first(); + } + + for (Integer page : sortedPagesWithHits) { + numberOfHitsPerPage.put(page, 0); //unknown number of matches in the page + pages.add(page); + currentHitPerPage.put(page, 0); //set current hit to 0th + } + } + + isPageInfoLoaded = true; + } + + @Override + public String getText() { + loadPageInfo(); //inits once + + String highLightField = LuceneQuery.HIGHLIGHT_FIELD_REGEX; + + SolrQuery q = new SolrQuery(); + q.setShowDebugInfo(DEBUG); //debug + String query = keywords.stream().map(keyword -> "/.*" + KeywordSearchUtil.escapeLuceneQuery(keyword) + ".*/").collect(Collectors.joining(" ")); + q.setQuery(LuceneQuery.HIGHLIGHT_FIELD_REGEX + ":" + query); + + String contentIdStr; + if (hasChunks) { + contentIdStr = solrObjectId + "_" + Integer.toString(this.currentPage); + } else { + contentIdStr = this.solrDocumentId; + } + + final String filterQuery = Server.Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(contentIdStr); + q.addFilterQuery(filterQuery); + q.addHighlightField(highLightField); //for exact highlighting, try content_ws field (with stored="true" in Solr schema) + + //tune the highlighter + q.setParam("hl.useFastVectorHighlighter", "true"); //fast highlighter scales better than standard one NON-NLS + q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only NON-NLS + q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only NON-NLS + q.setParam("hl.fragListBuilder", "single"); //makes sense for FastVectorHighlighter only NON-NLS + + //docs says makes sense for the original Highlighter only, but not really + q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //NON-NLS + + try { + QueryResponse response = solrServer.query(q, METHOD.POST); + Map>> responseHighlight = response.getHighlighting(); + + Map> responseHighlightID = responseHighlight.get(contentIdStr); + if (responseHighlightID == null) { + return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.getMarkup.noMatchMsg"); + } + List contentHighlights = responseHighlightID.get(highLightField); + if (contentHighlights == null) { + return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.getMarkup.noMatchMsg"); + } else { + // extracted content (minus highlight tags) is HTML-escaped + String highlightedContent = contentHighlights.get(0).trim(); + highlightedContent = insertAnchors(highlightedContent); + + return "
" + highlightedContent + "
"; //NON-NLS + } + } catch (Exception ex) { + LOGGER.log(Level.WARNING, "Error executing Solr highlighting query: " + keywords, ex); //NON-NLS + return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.getMarkup.queryFailedMsg"); + } + } + + @Override + public String toString() { + return getDisplayName(); + } + + @Override + public boolean isSearchable() { + return true; + } + + @Override + public String getAnchorPrefix() { + return ANCHOR_PREFIX; + } + + @Override + public int getNumberHits() { + if (!this.numberOfHitsPerPage.containsKey(this.currentPage)) { + return 0; + } + return this.numberOfHitsPerPage.get(this.currentPage); + } + + private String insertAnchors(String searchableContent) { + + final String insertPre = "$0"; //$0 will insert current regex match //NON-NLS + + Matcher m = Pattern.compile(HIGHLIGHT_PRE).matcher(searchableContent); + StringBuffer sb = new StringBuffer(searchableContent.length()); + int count; + for (count = 0; m.find(); count++) { + m.appendReplacement(sb, insertPre + count + insertPost); + } + m.appendTail(sb); + + //store total hits for this page, now that we know it + this.numberOfHitsPerPage.put(this.currentPage, count); + if (this.currentItem() == 0 && this.hasNextItem()) { + this.nextItem(); + } + + return sb.toString(); + } + + @Override + @Deprecated + // factory method to create an instance of this object + public AccountsText createInstance(long objectId, String keywordHitQuery, boolean isRegex, String originalQuery) { + return new AccountsText(String.valueOf(objectId), Collections.emptySet()); + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java index 0a4aeb7147..d9098731e4 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java @@ -98,7 +98,7 @@ public class ExtractedContentViewer implements DataContentViewer { Lookup nodeLookup = node.getLookup(); Content content = nodeLookup.lookup(Content.class); - Collection artifacts = node.getLookup().lookupAll(BlackboardArtifact.class); + Collection artifacts = nodeLookup.lookupAll(BlackboardArtifact.class); /* * Assemble a collection of all of the indexed text "sources" associated @@ -193,50 +193,12 @@ public class ExtractedContentViewer implements DataContentViewer { } return rawArtifactText; } -// private static TextMarkupLookup getHighlightLookup(BlackboardArtifact artifact, Content content) { -// if (artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() -// && artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT.getTypeID()) { -// return null; -// } -// -// long objectId = content.getId(); -// -// Lookup lookup = Lookup.getDefault(); -// TextMarkupLookup highlightFactory = lookup.lookup(TextMarkupLookup.class); -// try { -// List attributes = artifact.getAttributes(); -// String keyword = null; -// String regexp = null; -// boolean isRegexp = false; -// for (BlackboardAttribute att : attributes) { -// final int attributeTypeID = att.getAttributeType().getTypeID(); -// if (attributeTypeID == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()) { -// keyword = att.getValueString(); -// } else if (attributeTypeID == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER.getTypeID()) { -// keyword = att.getValueString(); -// isRegexp = true; -// } else if (attributeTypeID == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()) { -// regexp = att.getValueString(); -// isRegexp = StringUtils.isNotBlank(regexp); -// } else if (attributeTypeID == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT.getTypeID()) { -// objectId = att.getValueLong(); -// } -// } -// if (keyword != null) { -// String origQuery = isRegexp ? regexp : keyword; -// return highlightFactory.createInstance(objectId, keyword, isRegexp, origQuery); -// } -// } catch (TskCoreException ex) { -// LOGGER.log(Level.WARNING, "Failed to retrieve Blackboard Attributes", ex); //NON-NLS -// } -// return null; -// } @NbBundle.Messages({ "ExtractedContentViewer.creditCardNumber=Credit Card Number", "ExtractedContentViewer.creditCardNumbers=Credit Card Numbers"}) - private HighlightedText addAccountHighlightedText(Collection artifacts, @NotNull Content content) { - long objectId = content.getId(); + private AccountsText addAccountHighlightedText(Collection artifacts, @NotNull Content content) { + String objectId = String.valueOf(content.getId()); Set keywords = new HashSet<>(); try { if (artifacts == null || artifacts.isEmpty()) { @@ -244,6 +206,14 @@ public class ExtractedContentViewer implements DataContentViewer { } for (BlackboardArtifact artifact : artifacts) { try { + BlackboardAttribute solrIDAttr = artifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_SOLR_DOCUMENT_ID)); + if (solrIDAttr != null) { + String valueString = solrIDAttr.getValueString(); + if (StringUtils.isNotBlank(valueString)) { + objectId = valueString; + } + } + BlackboardAttribute keyWordAttr = artifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER)); if (keyWordAttr != null) { String valueString = keyWordAttr.getValueString(); @@ -261,7 +231,7 @@ public class ExtractedContentViewer implements DataContentViewer { } } if (keywords.isEmpty() == false) { - HighlightedText highlightedAccountText = new HighlightedText(objectId, String.join(" ", keywords), true); + AccountsText highlightedAccountText = new AccountsText(objectId,keywords); highlightedAccountText.setDisplayName(keywords.size() == 1 ? Bundle.ExtractedContentViewer_creditCardNumber() : Bundle.ExtractedContentViewer_creditCardNumbers()); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java index 965df03ece..4e4b55d5f7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java @@ -23,20 +23,17 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeSet; import java.util.logging.Level; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.lang.StringUtils; + +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest.METHOD; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.datamodel.TextMarkupLookup; +import org.sleuthkit.autopsy.keywordsearch.KeywordQueryFilter.FilterType; /** * Highlights hits for a given document. Knows about pages and such for the @@ -66,19 +63,6 @@ class HighlightedText implements IndexedText, TextMarkupLookup { private String originalQuery = null; //or original query if hits are not available private boolean isPageInfoLoaded = false; private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT); - private String displayName; - - synchronized String getDisplayName() { - if (StringUtils.isBlank(displayName)) { - return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.toString"); - } else { - return displayName; - } - } - - synchronized void setDisplayName(String displayName) { - this.displayName = displayName; - } HighlightedText(long objectId, String keywordHitQuery, boolean isRegex) { this.objectId = objectId; @@ -92,6 +76,8 @@ class HighlightedText implements IndexedText, TextMarkupLookup { this.solrServer = KeywordSearch.getServer(); this.numberPages = 0; this.currentPage = 0; + //hits are unknown + } //when the results are not known and need to requery to get hits @@ -136,7 +122,6 @@ class HighlightedText implements IndexedText, TextMarkupLookup { //if has chunks, get pages with hits if (hasChunks) { - TreeSet pagesSorted = new TreeSet<>(); //extract pages of interest, sorted /* @@ -144,66 +129,33 @@ class HighlightedText implements IndexedText, TextMarkupLookup { * need to perform the search to get the highlights. */ if (hits == null) { - - String highLightField = LuceneQuery.HIGHLIGHT_FIELD_REGEX; - String query; + String queryStr = KeywordSearchUtil.escapeLuceneQuery(this.keywordHitQuery); if (isRegex) { - String[] keywords = keywordHitQuery.split(" "); - query = Stream.of(keywords).map((String t) -> "/.*" + t + ".*/").collect(Collectors.joining(" ")); - } else { - query = keywordHitQuery; + //use white-space sep. field to get exact matches only of regex query result + queryStr = Server.Schema.CONTENT_WS + ":" + "\"" + queryStr + "\""; } - SolrQuery q = new SolrQuery(); - q.setShowDebugInfo(DEBUG); //debug - // input query has already been properly constructed and escaped - q.setQuery(highLightField + ":" + query); - q.setFields("id"); - q.addFilterQuery(Server.Schema.ID.toString() + ":" + this.objectId + "_*"); + Keyword keywordQuery = new Keyword(queryStr, !isRegex); + List keywords = new ArrayList<>(); + keywords.add(keywordQuery); + KeywordSearchQuery chunksQuery = new LuceneQuery(new KeywordList(keywords), keywordQuery); -// //tune the highlighter -// q.addHighlightField(highLightField); //for exact highlighting, try content_ws field (with stored="true" in Solr schema) -// q.setParam("hl.useFastVectorHighlighter", "true"); //fast highlighter scales better than standard one NON-NLS -// q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only NON-NLS -// q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only NON-NLS -// q.setParam("hl.fragListBuilder", "single"); //makes sense for FastVectorHighlighter only NON-NLS - //docs says makes sense for the original Highlighter only, but not really -// q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //NON-NLS + chunksQuery.addFilter(new KeywordQueryFilter(FilterType.CHUNK, this.objectId)); try { - QueryResponse response = solrServer.query(q, METHOD.POST); - - Set docs = LuceneQuery.filterOneHitPerDocument(response.getResults()); - for (SolrDocument resultDoc : docs) { - final String solrDocumentId = resultDoc.getFieldValue(Server.Schema.ID.toString()).toString(); - /** - * Parse the Solr document id to get the Solr object id - * and chunk id. The Solr object id will either be a - * file id or an artifact id from the case database. - * - * For every object (file or artifact) there will at - * least two Solr documents. One contains object - * metadata (chunk #1) and the second and subsequent - * documents contain chunks of the text. - */ - final int separatorIndex = solrDocumentId.indexOf(Server.ID_CHUNK_SEP); - if (-1 != separatorIndex) { - pagesSorted.add(Integer.parseInt(solrDocumentId.substring(separatorIndex + 1))); - } else { - pagesSorted.add(0); - } - } - - } catch (KeywordSearchModuleException | NoOpenCoreException | NumberFormatException ex) { - logger.log(Level.WARNING, "Error executing Solr highlighting query: " + keywordHitQuery, ex); //NON-NLS + hits = chunksQuery.performQuery(); + } catch (NoOpenCoreException ex) { + logger.log(Level.INFO, "Could not get chunk info and get highlights", ex); //NON-NLS + return; } + } - } else { - for (Keyword k : hits.getKeywords()) { - for (KeywordHit hit : hits.getResults(k)) { - int chunkID = hit.getChunkId(); - if (chunkID != 0 && this.objectId == hit.getSolrObjectId()) { - pagesSorted.add(chunkID); - } + //organize the hits by page, filter as needed + TreeSet pagesSorted = new TreeSet<>(); + for (Keyword k : hits.getKeywords()) { + for (KeywordHit hit : hits.getResults(k)) { + int chunkID = hit.getChunkId(); + if (chunkID != 0 && this.objectId == hit.getSolrObjectId()) { + pagesSorted.add(chunkID); } } } @@ -308,7 +260,8 @@ class HighlightedText implements IndexedText, TextMarkupLookup { @Override public int nextItem() { if (!hasNextItem()) { - throw new IllegalStateException(NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.nextItem.exception.msg")); + throw new IllegalStateException( + NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.nextItem.exception.msg")); } int cur = pagesToHits.get(currentPage) + 1; pagesToHits.put(currentPage, cur); @@ -318,7 +271,8 @@ class HighlightedText implements IndexedText, TextMarkupLookup { @Override public int previousItem() { if (!hasPreviousItem()) { - throw new IllegalStateException(NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.previousItem.exception.msg")); + throw new IllegalStateException( + NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.previousItem.exception.msg")); } int cur = pagesToHits.get(currentPage) - 1; pagesToHits.put(currentPage, cur); @@ -342,19 +296,19 @@ class HighlightedText implements IndexedText, TextMarkupLookup { public String getText() { loadPageInfo(); //inits once - String highLightField = LuceneQuery.HIGHLIGHT_FIELD_REGEX; - String query; + String highLightField = null; + if (isRegex) { - String[] keywords = keywordHitQuery.split(" "); - query = Stream.of(keywords).map((String t) -> "/.*" + t + ".*/").collect(Collectors.joining(" ")); + highLightField = LuceneQuery.HIGHLIGHT_FIELD_REGEX; } else { - query = keywordHitQuery; + highLightField = LuceneQuery.HIGHLIGHT_FIELD_LITERAL; } SolrQuery q = new SolrQuery(); q.setShowDebugInfo(DEBUG); //debug + // input query has already been properly constructed and escaped - q.setQuery(query); + q.setQuery(keywordHitQuery); String contentIdStr = Long.toString(this.objectId); if (hasChunks) { @@ -365,8 +319,12 @@ class HighlightedText implements IndexedText, TextMarkupLookup { q.addFilterQuery(filterQuery); q.addHighlightField(highLightField); //for exact highlighting, try content_ws field (with stored="true" in Solr schema) + //q.setHighlightSimplePre(HIGHLIGHT_PRE); //original highlighter only + //q.setHighlightSimplePost(HIGHLIGHT_POST); //original highlighter only + q.setHighlightFragsize(0); // don't fragment the highlight, works with original highlighter, or needs "single" list builder with FVH + //tune the highlighter - q.setParam("hl.useFastVectorHighlighter", "true"); //fast highlighter scales better than standard one NON-NLS + q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one NON-NLS q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only NON-NLS q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only NON-NLS q.setParam("hl.fragListBuilder", "single"); //makes sense for FastVectorHighlighter only NON-NLS @@ -400,7 +358,7 @@ class HighlightedText implements IndexedText, TextMarkupLookup { @Override public String toString() { - return getDisplayName(); + return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.toString"); } @Override